2024-11-15 16:57:47,399 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 16:57:47,414 main DEBUG Took 0.013032 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 16:57:47,415 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 16:57:47,415 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 16:57:47,416 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 16:57:47,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,427 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 16:57:47,440 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,442 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,444 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,445 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,446 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,448 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,449 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,450 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,451 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,451 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,452 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,453 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,454 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 16:57:47,455 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,456 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 16:57:47,458 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 16:57:47,460 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 16:57:47,462 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 16:57:47,463 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 16:57:47,464 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 16:57:47,465 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 16:57:47,476 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 16:57:47,480 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 16:57:47,482 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 16:57:47,482 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 16:57:47,483 main DEBUG createAppenders(={Console}) 2024-11-15 16:57:47,483 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 16:57:47,484 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 16:57:47,484 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 16:57:47,485 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 16:57:47,485 main DEBUG OutputStream closed 2024-11-15 16:57:47,486 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 16:57:47,486 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 16:57:47,487 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 16:57:47,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 16:57:47,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 16:57:47,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 16:57:47,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 16:57:47,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 16:57:47,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 16:57:47,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 16:57:47,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 16:57:47,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 16:57:47,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 16:57:47,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 16:57:47,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 16:57:47,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 16:57:47,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 16:57:47,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 16:57:47,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 16:57:47,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 16:57:47,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 16:57:47,567 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 16:57:47,567 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 16:57:47,567 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 16:57:47,568 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T16:57:47,790 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492 2024-11-15 16:57:47,793 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 16:57:47,793 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T16:57:47,802 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-15T16:57:47,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T16:57:47,835 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06, deleteOnExit=true 2024-11-15T16:57:47,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T16:57:47,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/test.cache.data in system properties and HBase conf 2024-11-15T16:57:47,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T16:57:47,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/hadoop.log.dir in system properties and HBase conf 2024-11-15T16:57:47,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T16:57:47,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T16:57:47,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T16:57:47,928 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T16:57:48,025 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T16:57:48,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T16:57:48,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T16:57:48,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T16:57:48,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T16:57:48,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T16:57:48,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T16:57:48,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T16:57:48,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T16:57:48,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T16:57:48,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/nfs.dump.dir in system properties and HBase conf 2024-11-15T16:57:48,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/java.io.tmpdir in system properties and HBase conf 2024-11-15T16:57:48,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T16:57:48,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T16:57:48,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T16:57:48,846 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T16:57:48,933 INFO [Time-limited test {}] log.Log(170): Logging initialized @2217ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T16:57:49,009 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:57:49,068 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T16:57:49,090 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T16:57:49,090 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T16:57:49,092 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T16:57:49,106 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:57:49,110 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/hadoop.log.dir/,AVAILABLE} 2024-11-15T16:57:49,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T16:57:49,316 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/java.io.tmpdir/jetty-localhost-40045-hadoop-hdfs-3_4_1-tests_jar-_-any-14963777931274913188/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T16:57:49,323 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40045} 2024-11-15T16:57:49,323 INFO [Time-limited test {}] server.Server(415): Started @2609ms 2024-11-15T16:57:49,721 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T16:57:49,728 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T16:57:49,729 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T16:57:49,729 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T16:57:49,730 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T16:57:49,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/hadoop.log.dir/,AVAILABLE} 2024-11-15T16:57:49,731 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T16:57:49,852 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/java.io.tmpdir/jetty-localhost-44709-hadoop-hdfs-3_4_1-tests_jar-_-any-8520462461095049031/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T16:57:49,853 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:44709} 2024-11-15T16:57:49,853 INFO [Time-limited test {}] server.Server(415): Started @3138ms 2024-11-15T16:57:49,910 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T16:57:50,365 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/data/data2/current/BP-433987452-172.17.0.2-1731689868601/current, will proceed with Du for space computation calculation, 2024-11-15T16:57:50,365 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/data/data1/current/BP-433987452-172.17.0.2-1731689868601/current, will proceed with Du for space computation calculation, 2024-11-15T16:57:50,409 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T16:57:50,461 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x95c723c077d07161 with lease ID 0x97b242cdad8dbd1a: Processing first storage report for DS-e85e10dc-25d3-4769-89cd-d5cabd645427 from datanode DatanodeRegistration(127.0.0.1:38441, datanodeUuid=75b1e1ef-6e8d-47b6-97f4-ab8c7454f9d7, infoPort=38217, infoSecurePort=0, ipcPort=43217, storageInfo=lv=-57;cid=testClusterID;nsid=1480960482;c=1731689868601) 2024-11-15T16:57:50,463 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x95c723c077d07161 with lease ID 0x97b242cdad8dbd1a: from storage DS-e85e10dc-25d3-4769-89cd-d5cabd645427 node DatanodeRegistration(127.0.0.1:38441, datanodeUuid=75b1e1ef-6e8d-47b6-97f4-ab8c7454f9d7, infoPort=38217, infoSecurePort=0, ipcPort=43217, storageInfo=lv=-57;cid=testClusterID;nsid=1480960482;c=1731689868601), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-15T16:57:50,463 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x95c723c077d07161 with lease ID 0x97b242cdad8dbd1a: Processing first storage report for DS-ec7e7f86-1044-46fd-8b0f-e6c115d94739 from datanode DatanodeRegistration(127.0.0.1:38441, datanodeUuid=75b1e1ef-6e8d-47b6-97f4-ab8c7454f9d7, infoPort=38217, infoSecurePort=0, ipcPort=43217, storageInfo=lv=-57;cid=testClusterID;nsid=1480960482;c=1731689868601) 2024-11-15T16:57:50,464 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x95c723c077d07161 with lease ID 0x97b242cdad8dbd1a: from storage DS-ec7e7f86-1044-46fd-8b0f-e6c115d94739 node DatanodeRegistration(127.0.0.1:38441, datanodeUuid=75b1e1ef-6e8d-47b6-97f4-ab8c7454f9d7, infoPort=38217, infoSecurePort=0, ipcPort=43217, storageInfo=lv=-57;cid=testClusterID;nsid=1480960482;c=1731689868601), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T16:57:50,501 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492 2024-11-15T16:57:50,576 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/zookeeper_0, clientPort=49383, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T16:57:50,585 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49383 2024-11-15T16:57:50,600 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:57:50,602 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:57:50,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741825_1001 (size=7) 2024-11-15T16:57:51,245 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a with version=8 2024-11-15T16:57:51,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/hbase-staging 2024-11-15T16:57:51,340 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T16:57:51,594 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:57:51,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:57:51,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:57:51,610 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:57:51,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:57:51,611 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:57:51,750 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T16:57:51,820 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T16:57:51,833 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T16:57:51,838 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:57:51,867 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7319 (auto-detected) 2024-11-15T16:57:51,868 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T16:57:51,886 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41785 2024-11-15T16:57:51,908 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41785 connecting to ZooKeeper ensemble=127.0.0.1:49383 2024-11-15T16:57:51,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:417850x0, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:57:51,941 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41785-0x101acddea460000 connected 2024-11-15T16:57:51,966 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:57:51,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:57:51,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:57:51,981 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a, hbase.cluster.distributed=false 2024-11-15T16:57:52,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:57:52,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41785 2024-11-15T16:57:52,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41785 2024-11-15T16:57:52,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41785 2024-11-15T16:57:52,018 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41785 2024-11-15T16:57:52,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41785 2024-11-15T16:57:52,130 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a9bde5b3a1b5:0 server-side Connection retries=45 2024-11-15T16:57:52,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:57:52,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T16:57:52,132 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T16:57:52,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T16:57:52,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T16:57:52,135 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T16:57:52,137 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T16:57:52,138 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35119 2024-11-15T16:57:52,140 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35119 connecting to ZooKeeper ensemble=127.0.0.1:49383 2024-11-15T16:57:52,141 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:57:52,144 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:57:52,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351190x0, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T16:57:52,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35119-0x101acddea460001 connected 2024-11-15T16:57:52,151 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T16:57:52,155 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T16:57:52,163 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T16:57:52,166 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T16:57:52,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T16:57:52,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35119 2024-11-15T16:57:52,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35119 2024-11-15T16:57:52,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35119 2024-11-15T16:57:52,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35119 2024-11-15T16:57:52,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35119 2024-11-15T16:57:52,191 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a9bde5b3a1b5:41785 2024-11-15T16:57:52,192 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:52,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:57:52,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:57:52,203 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:52,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:52,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T16:57:52,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:52,234 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T16:57:52,236 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a9bde5b3a1b5,41785,1731689871395 from backup master directory 2024-11-15T16:57:52,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:52,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:57:52,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T16:57:52,240 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:57:52,240 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:52,242 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T16:57:52,243 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T16:57:52,301 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/hbase.id] with ID: 62f8d877-313a-4ca6-8748-58e5181fa98a 2024-11-15T16:57:52,301 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/.tmp/hbase.id 2024-11-15T16:57:52,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741826_1002 (size=42) 2024-11-15T16:57:52,713 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/.tmp/hbase.id]:[hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/hbase.id] 2024-11-15T16:57:52,756 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T16:57:52,761 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T16:57:52,781 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-15T16:57:52,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:52,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:52,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741827_1003 (size=196) 2024-11-15T16:57:53,219 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:57:53,221 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T16:57:53,239 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:53,243 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:57:53,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741828_1004 (size=1189) 2024-11-15T16:57:53,292 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store 2024-11-15T16:57:53,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741829_1005 (size=34) 2024-11-15T16:57:53,716 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T16:57:53,719 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:57:53,720 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T16:57:53,720 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:57:53,720 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:57:53,722 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T16:57:53,722 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:57:53,722 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T16:57:53,724 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731689873720Disabling compacts and flushes for region at 1731689873720Disabling writes for close at 1731689873722 (+2 ms)Writing region close event to WAL at 1731689873722Closed at 1731689873722 2024-11-15T16:57:53,726 WARN [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/.initializing 2024-11-15T16:57:53,726 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/WALs/a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:53,735 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:57:53,749 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C41785%2C1731689871395, suffix=, logDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/WALs/a9bde5b3a1b5,41785,1731689871395, archiveDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/oldWALs, maxLogs=10 2024-11-15T16:57:53,773 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/WALs/a9bde5b3a1b5,41785,1731689871395/a9bde5b3a1b5%2C41785%2C1731689871395.1731689873754, exclude list is [], retry=0 2024-11-15T16:57:53,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38441,DS-e85e10dc-25d3-4769-89cd-d5cabd645427,DISK] 2024-11-15T16:57:53,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T16:57:53,837 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/WALs/a9bde5b3a1b5,41785,1731689871395/a9bde5b3a1b5%2C41785%2C1731689871395.1731689873754 2024-11-15T16:57:53,838 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38217:38217)] 2024-11-15T16:57:53,838 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:57:53,839 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:57:53,842 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,843 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,909 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T16:57:53,913 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:53,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:53,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T16:57:53,919 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:53,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:57:53,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T16:57:53,923 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:53,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:57:53,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T16:57:53,927 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:53,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:57:53,929 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,932 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,933 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,939 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,939 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,943 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T16:57:53,947 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T16:57:53,952 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:57:53,953 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72593843, jitterRate=0.08173255622386932}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T16:57:53,959 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731689873855Initializing all the Stores at 1731689873858 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689873858Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689873859 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689873859Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689873859Cleaning up temporary data from old regions at 1731689873940 (+81 ms)Region opened successfully at 1731689873959 (+19 ms) 2024-11-15T16:57:53,960 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T16:57:53,998 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1261aa41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:57:54,032 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T16:57:54,044 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T16:57:54,044 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T16:57:54,048 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T16:57:54,050 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T16:57:54,055 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T16:57:54,055 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T16:57:54,082 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T16:57:54,092 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T16:57:54,095 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T16:57:54,097 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T16:57:54,098 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T16:57:54,100 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T16:57:54,102 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T16:57:54,105 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T16:57:54,107 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T16:57:54,108 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T16:57:54,110 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T16:57:54,129 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T16:57:54,130 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T16:57:54,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:57:54,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:54,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T16:57:54,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:54,137 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a9bde5b3a1b5,41785,1731689871395, sessionid=0x101acddea460000, setting cluster-up flag (Was=false) 2024-11-15T16:57:54,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:54,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:54,157 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T16:57:54,159 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:54,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:54,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:54,172 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T16:57:54,174 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:54,181 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T16:57:54,267 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T16:57:54,278 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T16:57:54,281 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(746): ClusterId : 62f8d877-313a-4ca6-8748-58e5181fa98a 2024-11-15T16:57:54,284 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T16:57:54,286 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T16:57:54,291 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T16:57:54,291 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T16:57:54,294 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T16:57:54,292 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a9bde5b3a1b5,41785,1731689871395 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T16:57:54,295 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3d0853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a9bde5b3a1b5/172.17.0.2:0 2024-11-15T16:57:54,299 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:57:54,300 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:57:54,300 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:57:54,300 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=5, maxPoolSize=5 2024-11-15T16:57:54,300 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a9bde5b3a1b5:0, corePoolSize=10, maxPoolSize=10 2024-11-15T16:57:54,300 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,300 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:57:54,301 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,302 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731689904302 2024-11-15T16:57:54,304 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T16:57:54,306 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T16:57:54,307 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:57:54,307 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T16:57:54,310 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T16:57:54,310 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T16:57:54,310 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T16:57:54,311 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T16:57:54,311 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,314 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:54,314 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T16:57:54,315 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T16:57:54,317 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T16:57:54,317 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T16:57:54,318 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a9bde5b3a1b5:35119 2024-11-15T16:57:54,321 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T16:57:54,321 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T16:57:54,322 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T16:57:54,323 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T16:57:54,323 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T16:57:54,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741831_1007 (size=1321) 2024-11-15T16:57:54,326 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(2659): reportForDuty to master=a9bde5b3a1b5,41785,1731689871395 with port=35119, startcode=1731689872090 2024-11-15T16:57:54,330 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689874322,5,FailOnTimeoutGroup] 2024-11-15T16:57:54,331 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689874331,5,FailOnTimeoutGroup] 2024-11-15T16:57:54,331 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,331 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T16:57:54,333 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,334 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,337 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T16:57:54,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38615, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T16:57:54,377 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41785 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:54,379 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41785 {}] master.ServerManager(517): Registering regionserver=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:54,394 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T16:57:54,394 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45091 2024-11-15T16:57:54,394 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T16:57:54,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T16:57:54,399 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] zookeeper.ZKUtil(111): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:54,399 WARN [RS:0;a9bde5b3a1b5:35119 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T16:57:54,400 INFO [RS:0;a9bde5b3a1b5:35119 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:57:54,400 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/WALs/a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:54,402 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a9bde5b3a1b5,35119,1731689872090] 2024-11-15T16:57:54,428 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T16:57:54,443 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T16:57:54,449 INFO [RS:0;a9bde5b3a1b5:35119 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T16:57:54,449 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,450 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T16:57:54,457 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T16:57:54,458 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,459 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,459 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,459 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,459 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,459 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,459 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=2, maxPoolSize=2 2024-11-15T16:57:54,460 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,460 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,460 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,460 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,460 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,460 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a9bde5b3a1b5:0, corePoolSize=1, maxPoolSize=1 2024-11-15T16:57:54,461 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:57:54,461 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0, corePoolSize=3, maxPoolSize=3 2024-11-15T16:57:54,462 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,462 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,462 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,463 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,463 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,463 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,35119,1731689872090-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:57:54,484 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T16:57:54,486 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,35119,1731689872090-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,486 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,486 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.Replication(171): a9bde5b3a1b5,35119,1731689872090 started 2024-11-15T16:57:54,506 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:54,506 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1482): Serving as a9bde5b3a1b5,35119,1731689872090, RpcServer on a9bde5b3a1b5/172.17.0.2:35119, sessionid=0x101acddea460001 2024-11-15T16:57:54,507 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T16:57:54,507 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:54,508 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,35119,1731689872090' 2024-11-15T16:57:54,508 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T16:57:54,509 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T16:57:54,510 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T16:57:54,510 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T16:57:54,510 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:54,510 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a9bde5b3a1b5,35119,1731689872090' 2024-11-15T16:57:54,510 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T16:57:54,511 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T16:57:54,511 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T16:57:54,511 INFO [RS:0;a9bde5b3a1b5:35119 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T16:57:54,512 INFO [RS:0;a9bde5b3a1b5:35119 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T16:57:54,617 INFO [RS:0;a9bde5b3a1b5:35119 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T16:57:54,621 INFO [RS:0;a9bde5b3a1b5:35119 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C35119%2C1731689872090, suffix=, logDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/WALs/a9bde5b3a1b5,35119,1731689872090, archiveDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/oldWALs, maxLogs=32 2024-11-15T16:57:54,639 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/WALs/a9bde5b3a1b5,35119,1731689872090/a9bde5b3a1b5%2C35119%2C1731689872090.1731689874624, exclude list is [], retry=0 2024-11-15T16:57:54,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38441,DS-e85e10dc-25d3-4769-89cd-d5cabd645427,DISK] 2024-11-15T16:57:54,648 INFO [RS:0;a9bde5b3a1b5:35119 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/WALs/a9bde5b3a1b5,35119,1731689872090/a9bde5b3a1b5%2C35119%2C1731689872090.1731689874624 2024-11-15T16:57:54,649 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38217:38217)] 2024-11-15T16:57:54,728 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T16:57:54,728 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T16:57:54,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741833_1009 (size=32) 2024-11-15T16:57:55,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:57:55,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T16:57:55,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:57:55,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T16:57:55,149 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:57:55,150 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T16:57:55,153 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:57:55,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T16:57:55,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:57:55,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:57:55,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740 2024-11-15T16:57:55,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740 2024-11-15T16:57:55,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:57:55,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:57:55,163 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:57:55,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:57:55,170 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:57:55,170 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65620385, jitterRate=-0.022180065512657166}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:57:55,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731689875141Initializing all the Stores at 1731689875143 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689875143Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689875143Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689875143Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689875143Cleaning up temporary data from old regions at 1731689875163 (+20 ms)Region opened successfully at 1731689875172 (+9 ms) 2024-11-15T16:57:55,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T16:57:55,173 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T16:57:55,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T16:57:55,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T16:57:55,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T16:57:55,174 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T16:57:55,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731689875173Disabling compacts and flushes for region at 1731689875173Disabling writes for close at 1731689875173Writing region close event to WAL at 1731689875174 (+1 ms)Closed at 1731689875174 2024-11-15T16:57:55,178 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:57:55,178 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T16:57:55,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T16:57:55,193 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:57:55,196 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T16:57:55,349 DEBUG [a9bde5b3a1b5:41785 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T16:57:55,362 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:55,369 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,35119,1731689872090, state=OPENING 2024-11-15T16:57:55,374 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T16:57:55,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:55,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T16:57:55,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:57:55,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:57:55,379 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T16:57:55,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:57:55,557 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T16:57:55,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53895, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T16:57:55,573 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T16:57:55,573 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T16:57:55,573 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T16:57:55,576 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a9bde5b3a1b5%2C35119%2C1731689872090.meta, suffix=.meta, logDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/WALs/a9bde5b3a1b5,35119,1731689872090, archiveDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/oldWALs, maxLogs=32 2024-11-15T16:57:55,593 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/WALs/a9bde5b3a1b5,35119,1731689872090/a9bde5b3a1b5%2C35119%2C1731689872090.meta.1731689875578.meta, exclude list is [], retry=0 2024-11-15T16:57:55,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38441,DS-e85e10dc-25d3-4769-89cd-d5cabd645427,DISK] 2024-11-15T16:57:55,600 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/WALs/a9bde5b3a1b5,35119,1731689872090/a9bde5b3a1b5%2C35119%2C1731689872090.meta.1731689875578.meta 2024-11-15T16:57:55,600 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38217:38217)] 2024-11-15T16:57:55,601 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:57:55,603 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T16:57:55,606 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T16:57:55,611 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T16:57:55,615 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T16:57:55,616 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:57:55,616 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T16:57:55,616 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T16:57:55,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T16:57:55,621 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T16:57:55,621 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T16:57:55,624 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T16:57:55,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T16:57:55,626 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T16:57:55,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T16:57:55,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T16:57:55,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:55,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T16:57:55,629 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T16:57:55,631 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740 2024-11-15T16:57:55,634 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740 2024-11-15T16:57:55,636 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T16:57:55,636 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T16:57:55,637 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:57:55,640 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T16:57:55,642 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66702956, jitterRate=-0.0060485005378723145}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:57:55,642 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T16:57:55,644 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731689875617Writing region info on filesystem at 1731689875617Initializing all the Stores at 1731689875619 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689875619Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689875620 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689875620Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731689875620Cleaning up temporary data from old regions at 1731689875636 (+16 ms)Running coprocessor post-open hooks at 1731689875642 (+6 ms)Region opened successfully at 1731689875644 (+2 ms) 2024-11-15T16:57:55,652 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731689875547 2024-11-15T16:57:55,665 DEBUG [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T16:57:55,665 INFO [RS_OPEN_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T16:57:55,667 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:55,668 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a9bde5b3a1b5,35119,1731689872090, state=OPEN 2024-11-15T16:57:55,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:57:55,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T16:57:55,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:57:55,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T16:57:55,674 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:55,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T16:57:55,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a9bde5b3a1b5,35119,1731689872090 in 294 msec 2024-11-15T16:57:55,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T16:57:55,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 496 msec 2024-11-15T16:57:55,687 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T16:57:55,687 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T16:57:55,715 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:55,717 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:55,743 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:55,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35747, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:55,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5430 sec 2024-11-15T16:57:55,767 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731689875766, completionTime=-1 2024-11-15T16:57:55,769 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T16:57:55,769 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T16:57:55,796 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T16:57:55,796 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731689935796 2024-11-15T16:57:55,797 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731689995797 2024-11-15T16:57:55,797 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-15T16:57:55,799 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41785,1731689871395-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:55,799 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41785,1731689871395-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:55,800 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41785,1731689871395-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:55,801 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a9bde5b3a1b5:41785, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:55,802 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:55,802 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:55,808 DEBUG [master/a9bde5b3a1b5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T16:57:55,850 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.610sec 2024-11-15T16:57:55,851 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T16:57:55,852 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T16:57:55,854 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T16:57:55,854 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T16:57:55,854 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T16:57:55,855 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41785,1731689871395-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T16:57:55,855 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41785,1731689871395-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T16:57:55,868 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T16:57:55,868 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T16:57:55,869 INFO [master/a9bde5b3a1b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a9bde5b3a1b5,41785,1731689871395-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T16:57:55,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:55,894 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T16:57:55,894 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T16:57:55,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:55,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:55,907 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:55,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109c0bbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:55,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:55,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:55,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:55,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:55,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:55,964 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=177, ProcessCount=11, AvailableMemoryMB=5360 2024-11-15T16:57:55,973 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:57:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64e9cfde 2024-11-15T16:57:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:57:55,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:57:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T16:57:55,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:57:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T16:57:56,002 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:57:56,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T16:57:56,005 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:56,008 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:57:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:57:56,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741835_1011 (size=959) 2024-11-15T16:57:56,023 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T16:57:56,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741836_1012 (size=53) 2024-11-15T16:57:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:57:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:57:56,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:57:56,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1195534959da145ee3aa98da7f864a98, disabling compactions & flushes 2024-11-15T16:57:56,434 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:56,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:56,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. after waiting 0 ms 2024-11-15T16:57:56,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:56,434 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:56,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1195534959da145ee3aa98da7f864a98: Waiting for close lock at 1731689876433Disabling compacts and flushes for region at 1731689876433Disabling writes for close at 1731689876434 (+1 ms)Writing region close event to WAL at 1731689876434Closed at 1731689876434 2024-11-15T16:57:56,436 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:57:56,442 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731689876437"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689876437"}]},"ts":"1731689876437"} 2024-11-15T16:57:56,447 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:57:56,449 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:57:56,452 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689876449"}]},"ts":"1731689876449"} 2024-11-15T16:57:56,456 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T16:57:56,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, ASSIGN}] 2024-11-15T16:57:56,460 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, ASSIGN 2024-11-15T16:57:56,462 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=false 2024-11-15T16:57:56,613 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1195534959da145ee3aa98da7f864a98, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:56,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, ASSIGN because future has completed 2024-11-15T16:57:56,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:57:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:57:56,782 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:56,782 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:57:56,783 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,783 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:57:56,783 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,783 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,786 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,788 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1195534959da145ee3aa98da7f864a98 columnFamilyName A 2024-11-15T16:57:56,788 DEBUG [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:56,789 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] regionserver.HStore(327): Store=1195534959da145ee3aa98da7f864a98/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:57:56,789 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,792 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1195534959da145ee3aa98da7f864a98 columnFamilyName B 2024-11-15T16:57:56,792 DEBUG [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:56,793 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] regionserver.HStore(327): Store=1195534959da145ee3aa98da7f864a98/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:57:56,793 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,795 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1195534959da145ee3aa98da7f864a98 columnFamilyName C 2024-11-15T16:57:56,796 DEBUG [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:57:56,796 INFO [StoreOpener-1195534959da145ee3aa98da7f864a98-1 {}] regionserver.HStore(327): Store=1195534959da145ee3aa98da7f864a98/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:57:56,797 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,798 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,799 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,801 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,801 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,802 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:57:56,804 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,808 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:57:56,809 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1195534959da145ee3aa98da7f864a98; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73074976, jitterRate=0.08890199661254883}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:57:56,809 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:56,810 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1195534959da145ee3aa98da7f864a98: Running coprocessor pre-open hook at 1731689876783Writing region info on filesystem at 1731689876783Initializing all the Stores at 1731689876785 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689876785Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689876785Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689876786 (+1 ms)Cleaning up temporary data from old regions at 1731689876801 (+15 ms)Running coprocessor post-open hooks at 1731689876809 (+8 ms)Region opened successfully at 1731689876810 (+1 ms) 2024-11-15T16:57:56,812 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., pid=6, masterSystemTime=1731689876774 2024-11-15T16:57:56,816 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:56,816 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:56,817 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1195534959da145ee3aa98da7f864a98, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:56,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:57:56,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T16:57:56,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 in 203 msec 2024-11-15T16:57:56,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T16:57:56,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, ASSIGN in 369 msec 2024-11-15T16:57:56,832 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:57:56,832 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689876832"}]},"ts":"1731689876832"} 2024-11-15T16:57:56,835 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T16:57:56,837 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:57:56,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 843 msec 2024-11-15T16:57:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T16:57:57,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:57:57,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,156 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a99da4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,159 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e1d172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,161 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,168 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:57:57,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:57:57,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5effb952 2024-11-15T16:57:57,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:57:57,174 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:57:57,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,179 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75100a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,183 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f897912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b147913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,192 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408cf13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,196 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43659da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,199 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745237a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,203 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,206 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b12d099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,211 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63dc2eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332ec47f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,217 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d03237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,219 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24472a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,222 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552dc5bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,224 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e684afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,227 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288fb03e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,229 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c75040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:57:57,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:57:57,232 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:57:57,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:57:57,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:57:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:57:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:57:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:57:57,234 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:57:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aacb006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:57:57,239 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,239 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,239 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,240 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,242 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,243 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,248 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,253 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,260 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:57:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:57:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:57:57,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:57:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:57:57,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:57:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T16:57:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T16:57:57,288 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:57:57,292 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:57:57,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:57:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:57,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:57:57,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:57:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T16:57:57,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:57,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:57,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/ddc360dff76f4d6ba8fdf2f88cd122b0 is 50, key is test_row_0/A:col10/1731689877334/Put/seqid=0 2024-11-15T16:57:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741837_1013 (size=16681) 2024-11-15T16:57:57,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/ddc360dff76f4d6ba8fdf2f88cd122b0 2024-11-15T16:57:57,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689937519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689937519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689937527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689937536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689937536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T16:57:57,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/80b991b7c23b4b69a9a909bdb3159b04 is 50, key is test_row_0/B:col10/1731689877334/Put/seqid=0 2024-11-15T16:57:57,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741838_1014 (size=12001) 2024-11-15T16:57:57,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:57,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:57,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689937689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689937691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689937691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689937692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689937692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:57,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:57,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689937897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689937907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689937907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T16:57:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689937908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689937908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:57,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:57,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/80b991b7c23b4b69a9a909bdb3159b04 2024-11-15T16:57:58,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/1c138084c36f4c10bdabb1d3e15e7552 is 50, key is test_row_0/C:col10/1731689877334/Put/seqid=0 2024-11-15T16:57:58,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741839_1015 (size=12001) 2024-11-15T16:57:58,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689938216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689938226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689938227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689938227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689938228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:58,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:58,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T16:57:58,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:58,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:58,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/1c138084c36f4c10bdabb1d3e15e7552 2024-11-15T16:57:58,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/ddc360dff76f4d6ba8fdf2f88cd122b0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/ddc360dff76f4d6ba8fdf2f88cd122b0 2024-11-15T16:57:58,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/ddc360dff76f4d6ba8fdf2f88cd122b0, entries=250, sequenceid=15, filesize=16.3 K 2024-11-15T16:57:58,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/80b991b7c23b4b69a9a909bdb3159b04 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/80b991b7c23b4b69a9a909bdb3159b04 2024-11-15T16:57:58,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/80b991b7c23b4b69a9a909bdb3159b04, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T16:57:58,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/1c138084c36f4c10bdabb1d3e15e7552 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1c138084c36f4c10bdabb1d3e15e7552 2024-11-15T16:57:58,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1c138084c36f4c10bdabb1d3e15e7552, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T16:57:58,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:58,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1195534959da145ee3aa98da7f864a98 in 1261ms, sequenceid=15, compaction requested=false 2024-11-15T16:57:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T16:57:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:57:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:57:58,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T16:57:58,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e2e0d076df3248d78aff35a0e33eb8c7 is 50, key is test_row_0/A:col10/1731689877498/Put/seqid=0 2024-11-15T16:57:58,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741840_1016 (size=14341) 2024-11-15T16:57:58,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e2e0d076df3248d78aff35a0e33eb8c7 2024-11-15T16:57:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689938799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689938804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689938804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689938814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689938814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6 is 50, key is test_row_0/B:col10/1731689877498/Put/seqid=0 2024-11-15T16:57:58,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741841_1017 (size=12001) 2024-11-15T16:57:58,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6 2024-11-15T16:57:58,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/403965bf889d491bbaf3e0172abe6dc0 is 50, key is test_row_0/C:col10/1731689877498/Put/seqid=0 2024-11-15T16:57:58,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741842_1018 (size=12001) 2024-11-15T16:57:58,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/403965bf889d491bbaf3e0172abe6dc0 2024-11-15T16:57:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e2e0d076df3248d78aff35a0e33eb8c7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e2e0d076df3248d78aff35a0e33eb8c7 2024-11-15T16:57:58,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:58,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:58,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689938927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:57:58,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e2e0d076df3248d78aff35a0e33eb8c7, entries=200, sequenceid=38, filesize=14.0 K 2024-11-15T16:57:58,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6 2024-11-15T16:57:58,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T16:57:58,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689938936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/403965bf889d491bbaf3e0172abe6dc0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/403965bf889d491bbaf3e0172abe6dc0 2024-11-15T16:57:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689938937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689938938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689938938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:58,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/403965bf889d491bbaf3e0172abe6dc0, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T16:57:58,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 1195534959da145ee3aa98da7f864a98 in 230ms, sequenceid=38, compaction requested=false 2024-11-15T16:57:58,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:57:59,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T16:57:59,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:59,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T16:57:59,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a7492c2f05ef4f1a941e96c328c99c68 is 50, key is test_row_0/A:col10/1731689878800/Put/seqid=0 2024-11-15T16:57:59,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741843_1019 (size=9657) 2024-11-15T16:57:59,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:57:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:59,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689939228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689939233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689939236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689939238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689939241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689939347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689939347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689939356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689939357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689939357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T16:57:59,523 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a7492c2f05ef4f1a941e96c328c99c68 2024-11-15T16:57:59,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/1964518c989b4daf8b23c1900acb21b8 is 50, key is test_row_0/B:col10/1731689878800/Put/seqid=0 2024-11-15T16:57:59,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741844_1020 (size=9657) 2024-11-15T16:57:59,573 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/1964518c989b4daf8b23c1900acb21b8 2024-11-15T16:57:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689939577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689939577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689939577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689939577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689939579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/89992617cf94489a89e94a963821406e is 50, key is test_row_0/C:col10/1731689878800/Put/seqid=0 2024-11-15T16:57:59,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741845_1021 (size=9657) 2024-11-15T16:57:59,642 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/89992617cf94489a89e94a963821406e 2024-11-15T16:57:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a7492c2f05ef4f1a941e96c328c99c68 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a7492c2f05ef4f1a941e96c328c99c68 2024-11-15T16:57:59,672 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a7492c2f05ef4f1a941e96c328c99c68, entries=100, sequenceid=52, filesize=9.4 K 2024-11-15T16:57:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/1964518c989b4daf8b23c1900acb21b8 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/1964518c989b4daf8b23c1900acb21b8 2024-11-15T16:57:59,698 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/1964518c989b4daf8b23c1900acb21b8, entries=100, sequenceid=52, filesize=9.4 K 2024-11-15T16:57:59,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/89992617cf94489a89e94a963821406e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/89992617cf94489a89e94a963821406e 2024-11-15T16:57:59,718 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/89992617cf94489a89e94a963821406e, entries=100, sequenceid=52, filesize=9.4 K 2024-11-15T16:57:59,720 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1195534959da145ee3aa98da7f864a98 in 634ms, sequenceid=52, compaction requested=true 2024-11-15T16:57:59,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:57:59,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:57:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T16:57:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T16:57:59,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T16:57:59,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4330 sec 2024-11-15T16:57:59,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.4590 sec 2024-11-15T16:57:59,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:57:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:57:59,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T16:57:59,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/1c91fa4f5a6a479487069e8ef7be1c07 is 50, key is test_row_0/A:col10/1731689879890/Put/seqid=0 2024-11-15T16:57:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689939908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689939912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689939915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689939920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:57:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689939920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:57:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:57:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:57:59,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741846_1022 (size=12001) 2024-11-15T16:57:59,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/1c91fa4f5a6a479487069e8ef7be1c07 2024-11-15T16:57:59,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/3af7128b91c84c45ba1ef25e830f4189 is 50, key is test_row_0/B:col10/1731689879890/Put/seqid=0 2024-11-15T16:57:59,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741847_1023 (size=12001) 2024-11-15T16:57:59,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/3af7128b91c84c45ba1ef25e830f4189 2024-11-15T16:58:00,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/0f1a0cc430d44ce6b18aa9e1871fc94d is 50, key is test_row_0/C:col10/1731689879890/Put/seqid=0 2024-11-15T16:58:00,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741848_1024 (size=12001) 2024-11-15T16:58:00,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689940026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689940027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/0f1a0cc430d44ce6b18aa9e1871fc94d 2024-11-15T16:58:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689940027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689940036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689940037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/1c91fa4f5a6a479487069e8ef7be1c07 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1c91fa4f5a6a479487069e8ef7be1c07 2024-11-15T16:58:00,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1c91fa4f5a6a479487069e8ef7be1c07, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T16:58:00,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/3af7128b91c84c45ba1ef25e830f4189 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/3af7128b91c84c45ba1ef25e830f4189 2024-11-15T16:58:00,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/3af7128b91c84c45ba1ef25e830f4189, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T16:58:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/0f1a0cc430d44ce6b18aa9e1871fc94d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/0f1a0cc430d44ce6b18aa9e1871fc94d 2024-11-15T16:58:00,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/0f1a0cc430d44ce6b18aa9e1871fc94d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T16:58:00,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1195534959da145ee3aa98da7f864a98 in 208ms, sequenceid=76, compaction requested=true 2024-11-15T16:58:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:00,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:00,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:00,111 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:00,111 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:00,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:00,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:00,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:00,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:00,117 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:00,119 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:00,120 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:00,122 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/80b991b7c23b4b69a9a909bdb3159b04, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/1964518c989b4daf8b23c1900acb21b8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/3af7128b91c84c45ba1ef25e830f4189] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=44.6 K 2024-11-15T16:58:00,125 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:00,125 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:00,125 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:00,126 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/ddc360dff76f4d6ba8fdf2f88cd122b0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e2e0d076df3248d78aff35a0e33eb8c7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a7492c2f05ef4f1a941e96c328c99c68, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1c91fa4f5a6a479487069e8ef7be1c07] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=51.4 K 2024-11-15T16:58:00,126 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b991b7c23b4b69a9a909bdb3159b04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731689877334 2024-11-15T16:58:00,126 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc360dff76f4d6ba8fdf2f88cd122b0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731689877328 2024-11-15T16:58:00,127 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e1082bfd822c4bf8bc0ef1e9e55b2ca6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689877498 2024-11-15T16:58:00,127 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2e0d076df3248d78aff35a0e33eb8c7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689877498 2024-11-15T16:58:00,128 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7492c2f05ef4f1a941e96c328c99c68, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689878800 2024-11-15T16:58:00,128 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1964518c989b4daf8b23c1900acb21b8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689878800 2024-11-15T16:58:00,129 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c91fa4f5a6a479487069e8ef7be1c07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731689879222 2024-11-15T16:58:00,129 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af7128b91c84c45ba1ef25e830f4189, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731689879222 2024-11-15T16:58:00,189 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:00,190 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#12 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:00,190 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/0dd242fc74a44ff08c7650c7cafb7172 is 50, key is test_row_0/A:col10/1731689879890/Put/seqid=0 2024-11-15T16:58:00,192 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e9b6ce2ad17149cf936c87494a7855c0 is 50, key is test_row_0/B:col10/1731689879890/Put/seqid=0 2024-11-15T16:58:00,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741849_1025 (size=12139) 2024-11-15T16:58:00,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741850_1026 (size=12139) 2024-11-15T16:58:00,225 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/0dd242fc74a44ff08c7650c7cafb7172 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0dd242fc74a44ff08c7650c7cafb7172 2024-11-15T16:58:00,239 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e9b6ce2ad17149cf936c87494a7855c0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e9b6ce2ad17149cf936c87494a7855c0 2024-11-15T16:58:00,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:00,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T16:58:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:00,261 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into e9b6ce2ad17149cf936c87494a7855c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:00,261 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:00,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e8c669eaf860451696ee5253cfba75e3 is 50, key is test_row_0/A:col10/1731689880246/Put/seqid=0 2024-11-15T16:58:00,266 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into 0dd242fc74a44ff08c7650c7cafb7172(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:00,266 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:00,267 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=12, startTime=1731689880101; duration=0sec 2024-11-15T16:58:00,267 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=12, startTime=1731689880110; duration=0sec 2024-11-15T16:58:00,267 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:00,267 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:00,268 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:00,268 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:00,268 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:00,272 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:00,273 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:00,273 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:00,273 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1c138084c36f4c10bdabb1d3e15e7552, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/403965bf889d491bbaf3e0172abe6dc0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/89992617cf94489a89e94a963821406e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/0f1a0cc430d44ce6b18aa9e1871fc94d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=44.6 K 2024-11-15T16:58:00,275 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c138084c36f4c10bdabb1d3e15e7552, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731689877334 2024-11-15T16:58:00,276 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 403965bf889d491bbaf3e0172abe6dc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689877498 2024-11-15T16:58:00,280 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89992617cf94489a89e94a963821406e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689878800 2024-11-15T16:58:00,282 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1a0cc430d44ce6b18aa9e1871fc94d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731689879222 2024-11-15T16:58:00,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741851_1027 (size=14341) 2024-11-15T16:58:00,315 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#15 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:00,316 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/2235deaf87fa4f9b93ca0029dded7f6e is 50, key is test_row_0/C:col10/1731689879890/Put/seqid=0 2024-11-15T16:58:00,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689940307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689940307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689940314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689940316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689940320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741852_1028 (size=12139) 2024-11-15T16:58:00,359 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/2235deaf87fa4f9b93ca0029dded7f6e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/2235deaf87fa4f9b93ca0029dded7f6e 2024-11-15T16:58:00,374 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into 2235deaf87fa4f9b93ca0029dded7f6e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:00,374 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:00,378 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=12, startTime=1731689880112; duration=0sec 2024-11-15T16:58:00,378 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:00,378 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:00,421 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T16:58:00,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689940426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689940426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689940436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689940436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689940436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689940636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689940636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689940645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689940646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689940646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e8c669eaf860451696ee5253cfba75e3 2024-11-15T16:58:00,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/20e5970bdd2342e99e5e76f7f52d4cf7 is 50, key is test_row_0/B:col10/1731689880246/Put/seqid=0 2024-11-15T16:58:00,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741853_1029 (size=12001) 2024-11-15T16:58:00,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/20e5970bdd2342e99e5e76f7f52d4cf7 2024-11-15T16:58:00,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/ef68e567f8cc4aa8b36d7eb1407d80ff is 50, key is test_row_0/C:col10/1731689880246/Put/seqid=0 2024-11-15T16:58:00,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741854_1030 (size=12001) 2024-11-15T16:58:00,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689940946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689940947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689940965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689940966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689940966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:00,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,129 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:58:01,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/ef68e567f8cc4aa8b36d7eb1407d80ff 2024-11-15T16:58:01,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e8c669eaf860451696ee5253cfba75e3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e8c669eaf860451696ee5253cfba75e3 2024-11-15T16:58:01,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e8c669eaf860451696ee5253cfba75e3, entries=200, sequenceid=91, filesize=14.0 K 2024-11-15T16:58:01,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/20e5970bdd2342e99e5e76f7f52d4cf7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/20e5970bdd2342e99e5e76f7f52d4cf7 2024-11-15T16:58:01,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/20e5970bdd2342e99e5e76f7f52d4cf7, entries=150, sequenceid=91, filesize=11.7 K 2024-11-15T16:58:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/ef68e567f8cc4aa8b36d7eb1407d80ff as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ef68e567f8cc4aa8b36d7eb1407d80ff 2024-11-15T16:58:01,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ef68e567f8cc4aa8b36d7eb1407d80ff, entries=150, sequenceid=91, filesize=11.7 K 2024-11-15T16:58:01,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1195534959da145ee3aa98da7f864a98 in 981ms, sequenceid=91, compaction requested=false 2024-11-15T16:58:01,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T16:58:01,457 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:01,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T16:58:01,462 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:01,463 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T16:58:01,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T16:58:01,464 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:01,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:01,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/d53df0d3d184413abef30f92b1e7900e is 50, key is test_row_0/A:col10/1731689880310/Put/seqid=0 2024-11-15T16:58:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689941488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689941489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689941494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689941495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689941500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741855_1031 (size=12001) 2024-11-15T16:58:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T16:58:01,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689941606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689941608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689941616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T16:58:01,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:01,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689941616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:01,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689941617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T16:58:01,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T16:58:01,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:01,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T16:58:01,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T16:58:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689941816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689941816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,819 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T16:58:01,819 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T16:58:01,820 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T16:58:01,820 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T16:58:01,822 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T16:58:01,822 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T16:58:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689941826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689941826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689941836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:01,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/d53df0d3d184413abef30f92b1e7900e 2024-11-15T16:58:01,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T16:58:01,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:01,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:01,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:01,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:01,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0255b451b06341bca9ef9e7d7b53b418 is 50, key is test_row_0/B:col10/1731689880310/Put/seqid=0 2024-11-15T16:58:01,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741856_1032 (size=12001) 2024-11-15T16:58:01,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0255b451b06341bca9ef9e7d7b53b418 2024-11-15T16:58:01,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/5b527b36b4624b5389b9ed759299cb80 is 50, key is test_row_0/C:col10/1731689880310/Put/seqid=0 2024-11-15T16:58:02,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741857_1033 (size=12001) 2024-11-15T16:58:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T16:58:02,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T16:58:02,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:02,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689942126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689942126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689942136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689942136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689942146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T16:58:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:02,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/5b527b36b4624b5389b9ed759299cb80 2024-11-15T16:58:02,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T16:58:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:02,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/d53df0d3d184413abef30f92b1e7900e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d53df0d3d184413abef30f92b1e7900e 2024-11-15T16:58:02,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d53df0d3d184413abef30f92b1e7900e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T16:58:02,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0255b451b06341bca9ef9e7d7b53b418 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0255b451b06341bca9ef9e7d7b53b418 2024-11-15T16:58:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0255b451b06341bca9ef9e7d7b53b418, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T16:58:02,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/5b527b36b4624b5389b9ed759299cb80 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/5b527b36b4624b5389b9ed759299cb80 2024-11-15T16:58:02,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/5b527b36b4624b5389b9ed759299cb80, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T16:58:02,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1195534959da145ee3aa98da7f864a98 in 998ms, sequenceid=116, compaction requested=true 2024-11-15T16:58:02,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:02,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:02,463 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:02,463 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:02,465 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:02,465 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:02,465 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,465 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0dd242fc74a44ff08c7650c7cafb7172, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e8c669eaf860451696ee5253cfba75e3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d53df0d3d184413abef30f92b1e7900e] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=37.6 K 2024-11-15T16:58:02,466 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:02,466 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:02,466 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,466 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e9b6ce2ad17149cf936c87494a7855c0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/20e5970bdd2342e99e5e76f7f52d4cf7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0255b451b06341bca9ef9e7d7b53b418] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=35.3 K 2024-11-15T16:58:02,466 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd242fc74a44ff08c7650c7cafb7172, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731689879222 2024-11-15T16:58:02,467 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b6ce2ad17149cf936c87494a7855c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731689879222 2024-11-15T16:58:02,467 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c669eaf860451696ee5253cfba75e3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731689879914 2024-11-15T16:58:02,467 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e5970bdd2342e99e5e76f7f52d4cf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731689879914 2024-11-15T16:58:02,468 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53df0d3d184413abef30f92b1e7900e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731689880310 2024-11-15T16:58:02,468 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 0255b451b06341bca9ef9e7d7b53b418, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731689880310 2024-11-15T16:58:02,488 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:02,489 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e76d47b9b74649f7a78ba8cb5968f58f is 50, key is test_row_0/A:col10/1731689880310/Put/seqid=0 2024-11-15T16:58:02,490 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:02,491 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0c52d4d81d9a4735a0e3a41208ce72af is 50, key is test_row_0/B:col10/1731689880310/Put/seqid=0 2024-11-15T16:58:02,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741859_1035 (size=12241) 2024-11-15T16:58:02,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741858_1034 (size=12241) 2024-11-15T16:58:02,547 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0c52d4d81d9a4735a0e3a41208ce72af as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0c52d4d81d9a4735a0e3a41208ce72af 2024-11-15T16:58:02,548 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e76d47b9b74649f7a78ba8cb5968f58f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e76d47b9b74649f7a78ba8cb5968f58f 2024-11-15T16:58:02,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T16:58:02,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,563 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T16:58:02,565 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into e76d47b9b74649f7a78ba8cb5968f58f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:02,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:02,565 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=13, startTime=1731689882462; duration=0sec 2024-11-15T16:58:02,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:02,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:02,566 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:02,575 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into 0c52d4d81d9a4735a0e3a41208ce72af(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:02,575 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:02,576 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=13, startTime=1731689882463; duration=0sec 2024-11-15T16:58:02,576 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:02,576 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:02,576 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:02,576 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:02,576 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:02,576 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/2235deaf87fa4f9b93ca0029dded7f6e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ef68e567f8cc4aa8b36d7eb1407d80ff, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/5b527b36b4624b5389b9ed759299cb80] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=35.3 K 2024-11-15T16:58:02,577 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2235deaf87fa4f9b93ca0029dded7f6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731689879222 2024-11-15T16:58:02,578 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef68e567f8cc4aa8b36d7eb1407d80ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731689879914 2024-11-15T16:58:02,579 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b527b36b4624b5389b9ed759299cb80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731689880310 2024-11-15T16:58:02,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/5902f0ef44994e74950668b571006ab1 is 50, key is test_row_0/A:col10/1731689881491/Put/seqid=0 2024-11-15T16:58:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T16:58:02,599 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:02,599 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e3a2bda6efd7417ea831bff8a43d2ab6 is 50, key is test_row_0/C:col10/1731689880310/Put/seqid=0 2024-11-15T16:58:02,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741860_1036 (size=12051) 2024-11-15T16:58:02,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741861_1037 (size=12241) 2024-11-15T16:58:02,639 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e3a2bda6efd7417ea831bff8a43d2ab6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e3a2bda6efd7417ea831bff8a43d2ab6 2024-11-15T16:58:02,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:02,652 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into e3a2bda6efd7417ea831bff8a43d2ab6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:02,653 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:02,653 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=13, startTime=1731689882463; duration=0sec 2024-11-15T16:58:02,653 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:02,654 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:02,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689942674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689942681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689942682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689942684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689942684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689942796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689942796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689942796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689942796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689942796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689943005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689943006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689943006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689943006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689943006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,021 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/5902f0ef44994e74950668b571006ab1 2024-11-15T16:58:03,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/f98d9a156dd64611be514cd44967ee48 is 50, key is test_row_0/B:col10/1731689881491/Put/seqid=0 2024-11-15T16:58:03,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741862_1038 (size=12051) 2024-11-15T16:58:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689943316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689943316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689943316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689943317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689943317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,471 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/f98d9a156dd64611be514cd44967ee48 2024-11-15T16:58:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28 is 50, key is test_row_0/C:col10/1731689881491/Put/seqid=0 2024-11-15T16:58:03,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741863_1039 (size=12051) 2024-11-15T16:58:03,493 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28 2024-11-15T16:58:03,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/5902f0ef44994e74950668b571006ab1 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5902f0ef44994e74950668b571006ab1 2024-11-15T16:58:03,514 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5902f0ef44994e74950668b571006ab1, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T16:58:03,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/f98d9a156dd64611be514cd44967ee48 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/f98d9a156dd64611be514cd44967ee48 2024-11-15T16:58:03,524 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/f98d9a156dd64611be514cd44967ee48, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T16:58:03,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28 2024-11-15T16:58:03,538 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T16:58:03,539 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1195534959da145ee3aa98da7f864a98 in 976ms, sequenceid=131, compaction requested=false 2024-11-15T16:58:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T16:58:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T16:58:03,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T16:58:03,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0770 sec 2024-11-15T16:58:03,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0890 sec 2024-11-15T16:58:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T16:58:03,616 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:03,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T16:58:03,624 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T16:58:03,625 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:03,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T16:58:03,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T16:58:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:03,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T16:58:03,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/6b2a138f36894f26a49d288d5cc412ff is 50, key is test_row_0/A:col10/1731689882680/Put/seqid=0 2024-11-15T16:58:03,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741864_1040 (size=12151) 2024-11-15T16:58:03,821 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/6b2a138f36894f26a49d288d5cc412ff 2024-11-15T16:58:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/7d161bbcc5f64794977114f88d11caee is 50, key is test_row_0/B:col10/1731689882680/Put/seqid=0 2024-11-15T16:58:03,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689943842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689943843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689943845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689943843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689943850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741865_1041 (size=12151) 2024-11-15T16:58:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T16:58:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689943956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689943956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689943966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689943966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689943967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689944166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689944166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689944175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689944176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689944176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T16:58:04,263 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/7d161bbcc5f64794977114f88d11caee 2024-11-15T16:58:04,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/167274f15db548629343ffb07a089132 is 50, key is test_row_0/C:col10/1731689882680/Put/seqid=0 2024-11-15T16:58:04,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741866_1042 (size=12151) 2024-11-15T16:58:04,289 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/167274f15db548629343ffb07a089132 2024-11-15T16:58:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/6b2a138f36894f26a49d288d5cc412ff as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/6b2a138f36894f26a49d288d5cc412ff 2024-11-15T16:58:04,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/6b2a138f36894f26a49d288d5cc412ff, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T16:58:04,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/7d161bbcc5f64794977114f88d11caee as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7d161bbcc5f64794977114f88d11caee 2024-11-15T16:58:04,316 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7d161bbcc5f64794977114f88d11caee, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T16:58:04,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/167274f15db548629343ffb07a089132 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/167274f15db548629343ffb07a089132 2024-11-15T16:58:04,326 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/167274f15db548629343ffb07a089132, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T16:58:04,328 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1195534959da145ee3aa98da7f864a98 in 547ms, sequenceid=155, compaction requested=true 2024-11-15T16:58:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T16:58:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T16:58:04,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T16:58:04,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 709 msec 2024-11-15T16:58:04,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 722 msec 2024-11-15T16:58:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:04,496 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:04,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T16:58:04,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/3f602e7dbcd549f08702b9b264e4adea is 50, key is test_row_0/A:col10/1731689883842/Put/seqid=0 2024-11-15T16:58:04,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689944530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741867_1043 (size=12151) 2024-11-15T16:58:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/3f602e7dbcd549f08702b9b264e4adea 2024-11-15T16:58:04,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689944535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689944539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689944539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689944540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e68eafd6b89d49c8a5cc0c8708b83baa is 50, key is test_row_0/B:col10/1731689883842/Put/seqid=0 2024-11-15T16:58:04,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741868_1044 (size=12151) 2024-11-15T16:58:04,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689944646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689944656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689944656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689944656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689944666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T16:58:04,766 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:04,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T16:58:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T16:58:04,771 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:04,772 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:04,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689944856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689944866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689944866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T16:58:04,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689944876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689944876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:04,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T16:58:04,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:04,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:04,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:04,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:04,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:04,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e68eafd6b89d49c8a5cc0c8708b83baa 2024-11-15T16:58:04,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/c753f95e038c47a88ec2c478fb53c4aa is 50, key is test_row_0/C:col10/1731689883842/Put/seqid=0 2024-11-15T16:58:04,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741869_1045 (size=12151) 2024-11-15T16:58:05,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T16:58:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T16:58:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689945166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689945175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689945175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689945186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689945186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T16:58:05,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:05,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T16:58:05,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:05,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T16:58:05,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/c753f95e038c47a88ec2c478fb53c4aa 2024-11-15T16:58:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/3f602e7dbcd549f08702b9b264e4adea as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3f602e7dbcd549f08702b9b264e4adea 2024-11-15T16:58:05,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3f602e7dbcd549f08702b9b264e4adea, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T16:58:05,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/e68eafd6b89d49c8a5cc0c8708b83baa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e68eafd6b89d49c8a5cc0c8708b83baa 2024-11-15T16:58:05,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e68eafd6b89d49c8a5cc0c8708b83baa, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T16:58:05,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/c753f95e038c47a88ec2c478fb53c4aa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c753f95e038c47a88ec2c478fb53c4aa 2024-11-15T16:58:05,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c753f95e038c47a88ec2c478fb53c4aa, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T16:58:05,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 1195534959da145ee3aa98da7f864a98 in 947ms, sequenceid=172, compaction requested=true 2024-11-15T16:58:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:05,444 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:05,445 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:05,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:05,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:05,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:05,447 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,448 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e76d47b9b74649f7a78ba8cb5968f58f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5902f0ef44994e74950668b571006ab1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/6b2a138f36894f26a49d288d5cc412ff, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3f602e7dbcd549f08702b9b264e4adea] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=47.5 K 2024-11-15T16:58:05,448 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e76d47b9b74649f7a78ba8cb5968f58f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731689880310 2024-11-15T16:58:05,449 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:05,449 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:05,449 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,449 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0c52d4d81d9a4735a0e3a41208ce72af, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/f98d9a156dd64611be514cd44967ee48, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7d161bbcc5f64794977114f88d11caee, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e68eafd6b89d49c8a5cc0c8708b83baa] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=47.5 K 2024-11-15T16:58:05,449 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5902f0ef44994e74950668b571006ab1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731689881490 2024-11-15T16:58:05,450 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c52d4d81d9a4735a0e3a41208ce72af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731689880310 2024-11-15T16:58:05,450 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2a138f36894f26a49d288d5cc412ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689882678 2024-11-15T16:58:05,451 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f98d9a156dd64611be514cd44967ee48, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731689881490 2024-11-15T16:58:05,451 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f602e7dbcd549f08702b9b264e4adea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731689883838 2024-11-15T16:58:05,452 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d161bbcc5f64794977114f88d11caee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689882678 2024-11-15T16:58:05,453 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e68eafd6b89d49c8a5cc0c8708b83baa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731689883838 2024-11-15T16:58:05,470 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:05,471 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/5b5a1b317c8e438087a0ce1d036f9fdb is 50, key is test_row_0/A:col10/1731689883842/Put/seqid=0 2024-11-15T16:58:05,475 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:05,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/96fdbead822943ebb8ec3e79851ac9f6 is 50, key is test_row_0/B:col10/1731689883842/Put/seqid=0 2024-11-15T16:58:05,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741870_1046 (size=12527) 2024-11-15T16:58:05,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741871_1047 (size=12527) 2024-11-15T16:58:05,496 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/5b5a1b317c8e438087a0ce1d036f9fdb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5b5a1b317c8e438087a0ce1d036f9fdb 2024-11-15T16:58:05,502 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/96fdbead822943ebb8ec3e79851ac9f6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/96fdbead822943ebb8ec3e79851ac9f6 2024-11-15T16:58:05,506 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into 5b5a1b317c8e438087a0ce1d036f9fdb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:05,507 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:05,507 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=12, startTime=1731689885444; duration=0sec 2024-11-15T16:58:05,507 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:05,507 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:05,507 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:05,510 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:05,510 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:05,510 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,511 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e3a2bda6efd7417ea831bff8a43d2ab6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/167274f15db548629343ffb07a089132, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c753f95e038c47a88ec2c478fb53c4aa] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=47.5 K 2024-11-15T16:58:05,511 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3a2bda6efd7417ea831bff8a43d2ab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731689880310 2024-11-15T16:58:05,512 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36e5bd5ef1a43e3a6d6d2fa34ca0d28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731689881490 2024-11-15T16:58:05,512 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into 96fdbead822943ebb8ec3e79851ac9f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:05,512 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:05,512 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=12, startTime=1731689885444; duration=0sec 2024-11-15T16:58:05,512 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 167274f15db548629343ffb07a089132, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689882678 2024-11-15T16:58:05,512 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:05,512 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:05,513 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c753f95e038c47a88ec2c478fb53c4aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731689883838 2024-11-15T16:58:05,531 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:05,531 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/55f9e2959602457480f6c76b71ebb9ba is 50, key is test_row_0/C:col10/1731689883842/Put/seqid=0 2024-11-15T16:58:05,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741872_1048 (size=12527) 2024-11-15T16:58:05,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T16:58:05,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:05,545 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T16:58:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/bf9f15478e0541f58370a37da894b9aa is 50, key is test_row_0/A:col10/1731689884521/Put/seqid=0 2024-11-15T16:58:05,555 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/55f9e2959602457480f6c76b71ebb9ba as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/55f9e2959602457480f6c76b71ebb9ba 2024-11-15T16:58:05,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741873_1049 (size=12151) 2024-11-15T16:58:05,565 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into 55f9e2959602457480f6c76b71ebb9ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:05,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:05,565 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=12, startTime=1731689885445; duration=0sec 2024-11-15T16:58:05,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:05,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:05,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:05,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689945698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689945700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689945702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689945703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689945704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689945806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689945815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689945816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689945816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689945816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T16:58:05,961 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/bf9f15478e0541f58370a37da894b9aa 2024-11-15T16:58:05,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/93101a40b7634b7bb987049f033ac413 is 50, key is test_row_0/B:col10/1731689884521/Put/seqid=0 2024-11-15T16:58:05,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741874_1050 (size=12151) 2024-11-15T16:58:06,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689946016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689946025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689946025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689946026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689946026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689946325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689946336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689946336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689946336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689946336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,383 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/93101a40b7634b7bb987049f033ac413 2024-11-15T16:58:06,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/03d5f917a1ad43f5b1d92711b6b4f026 is 50, key is test_row_0/C:col10/1731689884521/Put/seqid=0 2024-11-15T16:58:06,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741875_1051 (size=12151) 2024-11-15T16:58:06,405 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/03d5f917a1ad43f5b1d92711b6b4f026 2024-11-15T16:58:06,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/bf9f15478e0541f58370a37da894b9aa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bf9f15478e0541f58370a37da894b9aa 2024-11-15T16:58:06,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bf9f15478e0541f58370a37da894b9aa, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T16:58:06,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/93101a40b7634b7bb987049f033ac413 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/93101a40b7634b7bb987049f033ac413 2024-11-15T16:58:06,430 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/93101a40b7634b7bb987049f033ac413, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T16:58:06,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/03d5f917a1ad43f5b1d92711b6b4f026 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/03d5f917a1ad43f5b1d92711b6b4f026 2024-11-15T16:58:06,439 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/03d5f917a1ad43f5b1d92711b6b4f026, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T16:58:06,441 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1195534959da145ee3aa98da7f864a98 in 896ms, sequenceid=194, compaction requested=false 2024-11-15T16:58:06,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T16:58:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T16:58:06,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T16:58:06,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6710 sec 2024-11-15T16:58:06,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6790 sec 2024-11-15T16:58:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:06,839 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:06,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T16:58:06,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/85fac4aacbd2481282ee6ee58f573484 is 50, key is test_row_0/A:col10/1731689886837/Put/seqid=0 2024-11-15T16:58:06,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741876_1052 (size=12151) 2024-11-15T16:58:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689946870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689946871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689946871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689946873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689946873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T16:58:06,916 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:06,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T16:58:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T16:58:06,919 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:06,921 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:06,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689946975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689946986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689946986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689946986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:06,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689946986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T16:58:07,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T16:58:07,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:07,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:07,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:07,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689947186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689947195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689947196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689947196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689947196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T16:58:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T16:58:07,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/85fac4aacbd2481282ee6ee58f573484 2024-11-15T16:58:07,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/15fb9f98e87f41538fb59eb9d07c8757 is 50, key is test_row_0/B:col10/1731689886837/Put/seqid=0 2024-11-15T16:58:07,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741877_1053 (size=12151) 2024-11-15T16:58:07,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/15fb9f98e87f41538fb59eb9d07c8757 2024-11-15T16:58:07,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e27d54168437474eb2fafd54593b8da9 is 50, key is test_row_0/C:col10/1731689886837/Put/seqid=0 2024-11-15T16:58:07,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741878_1054 (size=12151) 2024-11-15T16:58:07,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e27d54168437474eb2fafd54593b8da9 2024-11-15T16:58:07,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/85fac4aacbd2481282ee6ee58f573484 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/85fac4aacbd2481282ee6ee58f573484 2024-11-15T16:58:07,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/85fac4aacbd2481282ee6ee58f573484, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T16:58:07,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/15fb9f98e87f41538fb59eb9d07c8757 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/15fb9f98e87f41538fb59eb9d07c8757 2024-11-15T16:58:07,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/15fb9f98e87f41538fb59eb9d07c8757, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T16:58:07,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e27d54168437474eb2fafd54593b8da9 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e27d54168437474eb2fafd54593b8da9 2024-11-15T16:58:07,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e27d54168437474eb2fafd54593b8da9, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T16:58:07,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 1195534959da145ee3aa98da7f864a98 in 501ms, sequenceid=212, compaction requested=true 2024-11-15T16:58:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:07,341 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:07,341 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:07,342 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:07,342 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:07,342 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,343 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/96fdbead822943ebb8ec3e79851ac9f6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/93101a40b7634b7bb987049f033ac413, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/15fb9f98e87f41538fb59eb9d07c8757] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.0 K 2024-11-15T16:58:07,343 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:07,343 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fdbead822943ebb8ec3e79851ac9f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731689883838 2024-11-15T16:58:07,343 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:07,343 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,343 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5b5a1b317c8e438087a0ce1d036f9fdb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bf9f15478e0541f58370a37da894b9aa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/85fac4aacbd2481282ee6ee58f573484] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.0 K 2024-11-15T16:58:07,344 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 93101a40b7634b7bb987049f033ac413, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731689884521 2024-11-15T16:58:07,344 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b5a1b317c8e438087a0ce1d036f9fdb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731689883838 2024-11-15T16:58:07,344 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 15fb9f98e87f41538fb59eb9d07c8757, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731689885691 2024-11-15T16:58:07,345 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9f15478e0541f58370a37da894b9aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731689884521 2024-11-15T16:58:07,345 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85fac4aacbd2481282ee6ee58f573484, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731689885691 2024-11-15T16:58:07,360 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:07,361 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/691cde167e0840b9a97347e01e34e07e is 50, key is test_row_0/B:col10/1731689886837/Put/seqid=0 2024-11-15T16:58:07,364 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:07,365 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/bda3c4efd627452f82308d6843c287cf is 50, key is test_row_0/A:col10/1731689886837/Put/seqid=0 2024-11-15T16:58:07,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741879_1055 (size=12629) 2024-11-15T16:58:07,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T16:58:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,382 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T16:58:07,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741880_1056 (size=12629) 2024-11-15T16:58:07,391 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/691cde167e0840b9a97347e01e34e07e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/691cde167e0840b9a97347e01e34e07e 2024-11-15T16:58:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/39afb6ca361c4cbabd216a14b1267b69 is 50, key is test_row_0/A:col10/1731689886871/Put/seqid=0 2024-11-15T16:58:07,400 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into 691cde167e0840b9a97347e01e34e07e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:07,400 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:07,400 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=13, startTime=1731689887341; duration=0sec 2024-11-15T16:58:07,400 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:07,400 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:07,400 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:07,402 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:07,402 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:07,402 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:07,402 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/55f9e2959602457480f6c76b71ebb9ba, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/03d5f917a1ad43f5b1d92711b6b4f026, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e27d54168437474eb2fafd54593b8da9] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.0 K 2024-11-15T16:58:07,403 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f9e2959602457480f6c76b71ebb9ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731689883838 2024-11-15T16:58:07,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741881_1057 (size=12151) 2024-11-15T16:58:07,404 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d5f917a1ad43f5b1d92711b6b4f026, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731689884521 2024-11-15T16:58:07,405 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e27d54168437474eb2fafd54593b8da9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731689885691 2024-11-15T16:58:07,409 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/39afb6ca361c4cbabd216a14b1267b69 2024-11-15T16:58:07,437 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:07,437 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/ff14f00ec137411facc3c8678f75a854 is 50, key is test_row_0/C:col10/1731689886837/Put/seqid=0 2024-11-15T16:58:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/2100e6b8c9134104a2f5ba0fee7bf686 is 50, key is test_row_0/B:col10/1731689886871/Put/seqid=0 2024-11-15T16:58:07,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741882_1058 (size=12629) 2024-11-15T16:58:07,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741883_1059 (size=12151) 2024-11-15T16:58:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:07,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:07,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689947518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689947519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689947519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689947521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689947521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T16:58:07,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689947635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689947635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689947635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689947636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689947636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,794 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/bda3c4efd627452f82308d6843c287cf as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bda3c4efd627452f82308d6843c287cf 2024-11-15T16:58:07,803 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into bda3c4efd627452f82308d6843c287cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:07,803 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:07,803 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=13, startTime=1731689887341; duration=0sec 2024-11-15T16:58:07,803 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:07,803 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689947846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689947846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689947846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689947847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689947847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:07,860 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/2100e6b8c9134104a2f5ba0fee7bf686 2024-11-15T16:58:07,866 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/ff14f00ec137411facc3c8678f75a854 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ff14f00ec137411facc3c8678f75a854 2024-11-15T16:58:07,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/4707f5446d2b4bf0baae5c59c047ec71 is 50, key is test_row_0/C:col10/1731689886871/Put/seqid=0 2024-11-15T16:58:07,877 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into ff14f00ec137411facc3c8678f75a854(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:07,877 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:07,877 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=13, startTime=1731689887341; duration=0sec 2024-11-15T16:58:07,878 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:07,878 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:07,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741884_1060 (size=12151) 2024-11-15T16:58:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T16:58:08,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689948156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689948156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689948156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689948156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689948156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/4707f5446d2b4bf0baae5c59c047ec71 2024-11-15T16:58:08,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/39afb6ca361c4cbabd216a14b1267b69 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/39afb6ca361c4cbabd216a14b1267b69 2024-11-15T16:58:08,323 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/39afb6ca361c4cbabd216a14b1267b69, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T16:58:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/2100e6b8c9134104a2f5ba0fee7bf686 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2100e6b8c9134104a2f5ba0fee7bf686 2024-11-15T16:58:08,332 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2100e6b8c9134104a2f5ba0fee7bf686, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T16:58:08,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/4707f5446d2b4bf0baae5c59c047ec71 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/4707f5446d2b4bf0baae5c59c047ec71 2024-11-15T16:58:08,345 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/4707f5446d2b4bf0baae5c59c047ec71, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T16:58:08,348 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 1195534959da145ee3aa98da7f864a98 in 966ms, sequenceid=232, compaction requested=false 2024-11-15T16:58:08,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:08,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:08,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T16:58:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T16:58:08,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T16:58:08,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-15T16:58:08,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4390 sec 2024-11-15T16:58:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:08,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:08,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T16:58:08,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a25b0812584b4bc7ae82391beb9e2e20 is 50, key is test_row_0/A:col10/1731689887517/Put/seqid=0 2024-11-15T16:58:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689948692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741885_1061 (size=14541) 2024-11-15T16:58:08,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689948695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689948696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689948695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689948698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689948805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689948816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689948816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689948816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689948817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689949016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689949025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689949026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689949026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689949026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T16:58:09,066 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:09,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T16:58:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T16:58:09,070 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:09,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:09,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:09,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a25b0812584b4bc7ae82391beb9e2e20 2024-11-15T16:58:09,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/8fa00bebde584d6eb7758cd648c7cad2 is 50, key is test_row_0/B:col10/1731689887517/Put/seqid=0 2024-11-15T16:58:09,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741886_1062 (size=12151) 2024-11-15T16:58:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T16:58:09,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T16:58:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689949326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689949336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689949336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689949337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689949346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T16:58:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T16:58:09,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T16:58:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/8fa00bebde584d6eb7758cd648c7cad2 2024-11-15T16:58:09,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/9f37ebfeb6a8413b9f02d1a393e14742 is 50, key is test_row_0/C:col10/1731689887517/Put/seqid=0 2024-11-15T16:58:09,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741887_1063 (size=12151) 2024-11-15T16:58:09,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T16:58:09,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T16:58:09,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T16:58:09,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:09,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689949847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689949848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689949849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689949849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689949856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:09,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/9f37ebfeb6a8413b9f02d1a393e14742 2024-11-15T16:58:09,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a25b0812584b4bc7ae82391beb9e2e20 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a25b0812584b4bc7ae82391beb9e2e20 2024-11-15T16:58:09,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a25b0812584b4bc7ae82391beb9e2e20, entries=200, sequenceid=253, filesize=14.2 K 2024-11-15T16:58:09,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T16:58:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/8fa00bebde584d6eb7758cd648c7cad2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/8fa00bebde584d6eb7758cd648c7cad2 2024-11-15T16:58:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:09,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:10,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/8fa00bebde584d6eb7758cd648c7cad2, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T16:58:10,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/9f37ebfeb6a8413b9f02d1a393e14742 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/9f37ebfeb6a8413b9f02d1a393e14742 2024-11-15T16:58:10,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/9f37ebfeb6a8413b9f02d1a393e14742, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T16:58:10,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 1195534959da145ee3aa98da7f864a98 in 1343ms, sequenceid=253, compaction requested=true 2024-11-15T16:58:10,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:10,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:10,012 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:10,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:10,013 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:10,014 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:10,014 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:10,014 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:10,014 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:10,014 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:10,014 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:10,014 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bda3c4efd627452f82308d6843c287cf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/39afb6ca361c4cbabd216a14b1267b69, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a25b0812584b4bc7ae82391beb9e2e20] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=38.4 K 2024-11-15T16:58:10,014 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/691cde167e0840b9a97347e01e34e07e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2100e6b8c9134104a2f5ba0fee7bf686, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/8fa00bebde584d6eb7758cd648c7cad2] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.1 K 2024-11-15T16:58:10,015 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting bda3c4efd627452f82308d6843c287cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731689885691 2024-11-15T16:58:10,015 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 691cde167e0840b9a97347e01e34e07e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731689885691 2024-11-15T16:58:10,015 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39afb6ca361c4cbabd216a14b1267b69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731689886867 2024-11-15T16:58:10,015 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a25b0812584b4bc7ae82391beb9e2e20, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731689887517 2024-11-15T16:58:10,016 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2100e6b8c9134104a2f5ba0fee7bf686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731689886867 2024-11-15T16:58:10,017 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa00bebde584d6eb7758cd648c7cad2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731689887517 2024-11-15T16:58:10,028 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:10,030 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9acc169de2f543cf98e2c89bb9e0592f is 50, key is test_row_0/A:col10/1731689887517/Put/seqid=0 2024-11-15T16:58:10,032 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:10,032 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/ef8acd56c5bd4cd3af0d7f2d339cec8a is 50, key is test_row_0/B:col10/1731689887517/Put/seqid=0 2024-11-15T16:58:10,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741889_1065 (size=12731) 2024-11-15T16:58:10,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741888_1064 (size=12731) 2024-11-15T16:58:10,065 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/ef8acd56c5bd4cd3af0d7f2d339cec8a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ef8acd56c5bd4cd3af0d7f2d339cec8a 2024-11-15T16:58:10,067 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9acc169de2f543cf98e2c89bb9e0592f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9acc169de2f543cf98e2c89bb9e0592f 2024-11-15T16:58:10,077 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into ef8acd56c5bd4cd3af0d7f2d339cec8a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:10,077 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:10,077 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into 9acc169de2f543cf98e2c89bb9e0592f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:10,077 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=13, startTime=1731689890013; duration=0sec 2024-11-15T16:58:10,077 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:10,077 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=13, startTime=1731689890012; duration=0sec 2024-11-15T16:58:10,077 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:10,077 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:10,077 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:10,077 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:10,077 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:10,080 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:10,080 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:10,080 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:10,080 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ff14f00ec137411facc3c8678f75a854, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/4707f5446d2b4bf0baae5c59c047ec71, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/9f37ebfeb6a8413b9f02d1a393e14742] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.1 K 2024-11-15T16:58:10,081 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting ff14f00ec137411facc3c8678f75a854, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731689885691 2024-11-15T16:58:10,081 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 4707f5446d2b4bf0baae5c59c047ec71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731689886867 2024-11-15T16:58:10,082 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f37ebfeb6a8413b9f02d1a393e14742, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731689887517 2024-11-15T16:58:10,092 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:10,092 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/8e12bc8068f24725b863520a2c36a5c5 is 50, key is test_row_0/C:col10/1731689887517/Put/seqid=0 2024-11-15T16:58:10,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741890_1066 (size=12731) 2024-11-15T16:58:10,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T16:58:10,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:10,152 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T16:58:10,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/3426c2ba610c4cbe90918542aa702505 is 50, key is test_row_0/A:col10/1731689888695/Put/seqid=0 2024-11-15T16:58:10,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741891_1067 (size=12301) 2024-11-15T16:58:10,173 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/3426c2ba610c4cbe90918542aa702505 2024-11-15T16:58:10,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/7f9194bbac3a427686aa54f1f60f7d72 is 50, key is test_row_0/B:col10/1731689888695/Put/seqid=0 2024-11-15T16:58:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T16:58:10,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741892_1068 (size=12301) 2024-11-15T16:58:10,521 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/8e12bc8068f24725b863520a2c36a5c5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/8e12bc8068f24725b863520a2c36a5c5 2024-11-15T16:58:10,531 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into 8e12bc8068f24725b863520a2c36a5c5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:10,531 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:10,532 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=13, startTime=1731689890013; duration=0sec 2024-11-15T16:58:10,532 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:10,532 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:10,607 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/7f9194bbac3a427686aa54f1f60f7d72 2024-11-15T16:58:10,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e7734291685243fca8d302573f107fe5 is 50, key is test_row_0/C:col10/1731689888695/Put/seqid=0 2024-11-15T16:58:10,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741893_1069 (size=12301) 2024-11-15T16:58:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:10,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689950884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689950886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689950887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689950891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689950889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:10,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689950995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689950996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689950996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689951006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689951006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,024 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e7734291685243fca8d302573f107fe5 2024-11-15T16:58:11,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/3426c2ba610c4cbe90918542aa702505 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3426c2ba610c4cbe90918542aa702505 2024-11-15T16:58:11,040 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3426c2ba610c4cbe90918542aa702505, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T16:58:11,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/7f9194bbac3a427686aa54f1f60f7d72 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7f9194bbac3a427686aa54f1f60f7d72 2024-11-15T16:58:11,053 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7f9194bbac3a427686aa54f1f60f7d72, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T16:58:11,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e7734291685243fca8d302573f107fe5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e7734291685243fca8d302573f107fe5 2024-11-15T16:58:11,066 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e7734291685243fca8d302573f107fe5, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T16:58:11,067 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 1195534959da145ee3aa98da7f864a98 in 915ms, sequenceid=273, compaction requested=false 2024-11-15T16:58:11,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T16:58:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T16:58:11,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T16:58:11,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9990 sec 2024-11-15T16:58:11,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.0070 sec 2024-11-15T16:58:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:11,211 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:11,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T16:58:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T16:58:11,216 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:11,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T16:58:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T16:58:11,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:11,222 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:11,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:11,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/b5d89e54221f49d997ec0c7040a97a96 is 50, key is test_row_0/A:col10/1731689891209/Put/seqid=0 2024-11-15T16:58:11,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741894_1070 (size=12301) 2024-11-15T16:58:11,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/b5d89e54221f49d997ec0c7040a97a96 2024-11-15T16:58:11,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689951234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689951241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689951241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/a46b89bc7ce441b6a65fb82da8eb7c3c is 50, key is test_row_0/B:col10/1731689891209/Put/seqid=0 2024-11-15T16:58:11,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689951244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689951244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741895_1071 (size=12301) 2024-11-15T16:58:11,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/a46b89bc7ce441b6a65fb82da8eb7c3c 2024-11-15T16:58:11,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e087b6c49ae84baeaad95864156443bb is 50, key is test_row_0/C:col10/1731689891209/Put/seqid=0 2024-11-15T16:58:11,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741896_1072 (size=12301) 2024-11-15T16:58:11,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e087b6c49ae84baeaad95864156443bb 2024-11-15T16:58:11,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/b5d89e54221f49d997ec0c7040a97a96 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/b5d89e54221f49d997ec0c7040a97a96 2024-11-15T16:58:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T16:58:11,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/b5d89e54221f49d997ec0c7040a97a96, entries=150, sequenceid=294, filesize=12.0 K 2024-11-15T16:58:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/a46b89bc7ce441b6a65fb82da8eb7c3c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/a46b89bc7ce441b6a65fb82da8eb7c3c 2024-11-15T16:58:11,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/a46b89bc7ce441b6a65fb82da8eb7c3c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-15T16:58:11,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/e087b6c49ae84baeaad95864156443bb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e087b6c49ae84baeaad95864156443bb 2024-11-15T16:58:11,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e087b6c49ae84baeaad95864156443bb, entries=150, sequenceid=294, filesize=12.0 K 2024-11-15T16:58:11,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 1195534959da145ee3aa98da7f864a98 in 139ms, sequenceid=294, compaction requested=true 2024-11-15T16:58:11,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:11,351 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:11,351 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:11,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:11,353 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:11,353 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:11,353 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:11,353 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:11,353 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,353 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,353 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9acc169de2f543cf98e2c89bb9e0592f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3426c2ba610c4cbe90918542aa702505, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/b5d89e54221f49d997ec0c7040a97a96] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.5 K 2024-11-15T16:58:11,353 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ef8acd56c5bd4cd3af0d7f2d339cec8a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7f9194bbac3a427686aa54f1f60f7d72, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/a46b89bc7ce441b6a65fb82da8eb7c3c] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.5 K 2024-11-15T16:58:11,354 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9acc169de2f543cf98e2c89bb9e0592f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731689887517 2024-11-15T16:58:11,354 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting ef8acd56c5bd4cd3af0d7f2d339cec8a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731689887517 2024-11-15T16:58:11,354 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3426c2ba610c4cbe90918542aa702505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731689888692 2024-11-15T16:58:11,355 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f9194bbac3a427686aa54f1f60f7d72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731689888692 2024-11-15T16:58:11,355 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d89e54221f49d997ec0c7040a97a96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731689890878 2024-11-15T16:58:11,356 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a46b89bc7ce441b6a65fb82da8eb7c3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731689890878 2024-11-15T16:58:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:11,360 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:11,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T16:58:11,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9e1bf367874e4cb8a140d869a272035a is 50, key is test_row_0/A:col10/1731689891360/Put/seqid=0 2024-11-15T16:58:11,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T16:58:11,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:11,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,379 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#61 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:11,379 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/443b70d25aef4435af49aa21ec4f7f1e is 50, key is test_row_0/B:col10/1731689891209/Put/seqid=0 2024-11-15T16:58:11,394 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:11,395 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/5ae0805e9c784185a40ee917afe92e19 is 50, key is test_row_0/A:col10/1731689891209/Put/seqid=0 2024-11-15T16:58:11,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689951385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689951386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689951387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689951388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689951390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741897_1073 (size=17181) 2024-11-15T16:58:11,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9e1bf367874e4cb8a140d869a272035a 2024-11-15T16:58:11,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741898_1074 (size=12983) 2024-11-15T16:58:11,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741899_1075 (size=12983) 2024-11-15T16:58:11,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/4f4573886d324588929a014177a6d2b7 is 50, key is test_row_0/B:col10/1731689891360/Put/seqid=0 2024-11-15T16:58:11,436 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/443b70d25aef4435af49aa21ec4f7f1e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/443b70d25aef4435af49aa21ec4f7f1e 2024-11-15T16:58:11,436 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/5ae0805e9c784185a40ee917afe92e19 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5ae0805e9c784185a40ee917afe92e19 2024-11-15T16:58:11,447 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into 443b70d25aef4435af49aa21ec4f7f1e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:11,447 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:11,447 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into 5ae0805e9c784185a40ee917afe92e19(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:11,447 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=13, startTime=1731689891351; duration=0sec 2024-11-15T16:58:11,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:11,447 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=13, startTime=1731689891351; duration=0sec 2024-11-15T16:58:11,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:11,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:11,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:11,448 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:11,448 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:11,449 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:11,449 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:11,449 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,449 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/8e12bc8068f24725b863520a2c36a5c5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e7734291685243fca8d302573f107fe5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e087b6c49ae84baeaad95864156443bb] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.5 K 2024-11-15T16:58:11,450 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e12bc8068f24725b863520a2c36a5c5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731689887517 2024-11-15T16:58:11,451 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7734291685243fca8d302573f107fe5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731689888692 2024-11-15T16:58:11,451 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e087b6c49ae84baeaad95864156443bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731689890878 2024-11-15T16:58:11,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741900_1076 (size=12301) 2024-11-15T16:58:11,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/4f4573886d324588929a014177a6d2b7 2024-11-15T16:58:11,461 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:11,462 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f3c2eaaf6ad04fcd9f51881676146e3e is 50, key is test_row_0/C:col10/1731689891209/Put/seqid=0 2024-11-15T16:58:11,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741901_1077 (size=12983) 2024-11-15T16:58:11,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/686e22e4778545aeb4385f12f0fd74b4 is 50, key is test_row_0/C:col10/1731689891360/Put/seqid=0 2024-11-15T16:58:11,479 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f3c2eaaf6ad04fcd9f51881676146e3e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f3c2eaaf6ad04fcd9f51881676146e3e 2024-11-15T16:58:11,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741902_1078 (size=12301) 2024-11-15T16:58:11,488 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into f3c2eaaf6ad04fcd9f51881676146e3e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:11,488 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:11,488 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=13, startTime=1731689891351; duration=0sec 2024-11-15T16:58:11,488 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:11,489 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:11,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689951506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689951506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689951506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689951506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689951506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T16:58:11,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:11,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T16:58:11,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T16:58:11,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:11,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689951715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689951715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689951716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689951716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689951716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:11,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T16:58:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T16:58:11,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/686e22e4778545aeb4385f12f0fd74b4 2024-11-15T16:58:11,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9e1bf367874e4cb8a140d869a272035a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9e1bf367874e4cb8a140d869a272035a 2024-11-15T16:58:11,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9e1bf367874e4cb8a140d869a272035a, entries=250, sequenceid=313, filesize=16.8 K 2024-11-15T16:58:11,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/4f4573886d324588929a014177a6d2b7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/4f4573886d324588929a014177a6d2b7 2024-11-15T16:58:11,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/4f4573886d324588929a014177a6d2b7, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T16:58:11,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/686e22e4778545aeb4385f12f0fd74b4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/686e22e4778545aeb4385f12f0fd74b4 2024-11-15T16:58:11,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/686e22e4778545aeb4385f12f0fd74b4, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T16:58:11,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 1195534959da145ee3aa98da7f864a98 in 557ms, sequenceid=313, compaction requested=false 2024-11-15T16:58:11,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:11,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T16:58:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:11,991 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T16:58:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9abef813e3ef4110ba2059f8d64141e0 is 50, key is test_row_0/A:col10/1731689891385/Put/seqid=0 2024-11-15T16:58:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741903_1079 (size=12301) 2024-11-15T16:58:12,004 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9abef813e3ef4110ba2059f8d64141e0 2024-11-15T16:58:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/16c60302d07345faadf12d4a12207f8c is 50, key is test_row_0/B:col10/1731689891385/Put/seqid=0 2024-11-15T16:58:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:12,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:12,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741904_1080 (size=12301) 2024-11-15T16:58:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689952045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689952046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689952048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689952050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689952050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689952155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689952156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689952156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689952156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689952156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T16:58:12,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689952366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689952366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689952366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689952366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689952367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/16c60302d07345faadf12d4a12207f8c 2024-11-15T16:58:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f2f966f32e07483ab47bfd607b994180 is 50, key is test_row_0/C:col10/1731689891385/Put/seqid=0 2024-11-15T16:58:12,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741905_1081 (size=12301) 2024-11-15T16:58:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689952676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689952676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689952676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689952676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689952677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:12,851 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f2f966f32e07483ab47bfd607b994180 2024-11-15T16:58:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9abef813e3ef4110ba2059f8d64141e0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9abef813e3ef4110ba2059f8d64141e0 2024-11-15T16:58:12,864 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9abef813e3ef4110ba2059f8d64141e0, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:58:12,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/16c60302d07345faadf12d4a12207f8c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/16c60302d07345faadf12d4a12207f8c 2024-11-15T16:58:12,871 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/16c60302d07345faadf12d4a12207f8c, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:58:12,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f2f966f32e07483ab47bfd607b994180 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f2f966f32e07483ab47bfd607b994180 2024-11-15T16:58:12,877 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f2f966f32e07483ab47bfd607b994180, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:58:12,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 1195534959da145ee3aa98da7f864a98 in 887ms, sequenceid=333, compaction requested=true 2024-11-15T16:58:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T16:58:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T16:58:12,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T16:58:12,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6570 sec 2024-11-15T16:58:12,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6650 sec 2024-11-15T16:58:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:13,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:13,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T16:58:13,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/8a8ae3c2e649406596c6e57f89c3067a is 50, key is test_row_0/A:col10/1731689893187/Put/seqid=0 2024-11-15T16:58:13,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689953201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689953201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689953203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689953202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689953206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741906_1082 (size=12301) 2024-11-15T16:58:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689953328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689953328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689953328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689953329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689953328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T16:58:13,367 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:13,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T16:58:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T16:58:13,371 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:13,372 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:13,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T16:58:13,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:13,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689953546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689953546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689953546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689953546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689953546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/8a8ae3c2e649406596c6e57f89c3067a 2024-11-15T16:58:13,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/b340e54e30a24fea80c82f2bc692b4a1 is 50, key is test_row_0/B:col10/1731689893187/Put/seqid=0 2024-11-15T16:58:13,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741907_1083 (size=12301) 2024-11-15T16:58:13,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T16:58:13,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689953856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689953856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689953856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689953856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689953856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:13,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:13,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:13,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:13,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T16:58:14,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/b340e54e30a24fea80c82f2bc692b4a1 2024-11-15T16:58:14,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/da8d4fd93f534329a627f048d36bbce1 is 50, key is test_row_0/C:col10/1731689893187/Put/seqid=0 2024-11-15T16:58:14,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741908_1084 (size=12301) 2024-11-15T16:58:14,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689954365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689954365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689954365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689954366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689954366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:14,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/da8d4fd93f534329a627f048d36bbce1 2024-11-15T16:58:14,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:14,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:14,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/8a8ae3c2e649406596c6e57f89c3067a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/8a8ae3c2e649406596c6e57f89c3067a 2024-11-15T16:58:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:14,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/8a8ae3c2e649406596c6e57f89c3067a, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T16:58:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/b340e54e30a24fea80c82f2bc692b4a1 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/b340e54e30a24fea80c82f2bc692b4a1 2024-11-15T16:58:14,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/b340e54e30a24fea80c82f2bc692b4a1, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T16:58:14,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/da8d4fd93f534329a627f048d36bbce1 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/da8d4fd93f534329a627f048d36bbce1 2024-11-15T16:58:14,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/da8d4fd93f534329a627f048d36bbce1, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T16:58:14,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1195534959da145ee3aa98da7f864a98 in 1281ms, sequenceid=356, compaction requested=true 2024-11-15T16:58:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:14,471 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:14,471 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:14,473 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:14,473 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:14,473 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,473 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5ae0805e9c784185a40ee917afe92e19, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9e1bf367874e4cb8a140d869a272035a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9abef813e3ef4110ba2059f8d64141e0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/8a8ae3c2e649406596c6e57f89c3067a] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=53.5 K 2024-11-15T16:58:14,474 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:14,474 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:14,474 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,474 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/443b70d25aef4435af49aa21ec4f7f1e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/4f4573886d324588929a014177a6d2b7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/16c60302d07345faadf12d4a12207f8c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/b340e54e30a24fea80c82f2bc692b4a1] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=48.7 K 2024-11-15T16:58:14,474 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae0805e9c784185a40ee917afe92e19, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731689890878 2024-11-15T16:58:14,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 443b70d25aef4435af49aa21ec4f7f1e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731689890878 2024-11-15T16:58:14,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f4573886d324588929a014177a6d2b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731689891232 2024-11-15T16:58:14,476 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e1bf367874e4cb8a140d869a272035a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731689891232 2024-11-15T16:58:14,476 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c60302d07345faadf12d4a12207f8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689891380 2024-11-15T16:58:14,476 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9abef813e3ef4110ba2059f8d64141e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689891380 2024-11-15T16:58:14,476 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting b340e54e30a24fea80c82f2bc692b4a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731689892040 2024-11-15T16:58:14,477 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8ae3c2e649406596c6e57f89c3067a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731689892040 2024-11-15T16:58:14,490 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:14,491 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:14,491 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/cbeef73413b64174bf261bbb0d593284 is 50, key is test_row_0/A:col10/1731689893187/Put/seqid=0 2024-11-15T16:58:14,491 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/06c8523e6f6c49b2830458e597e3fc29 is 50, key is test_row_0/B:col10/1731689893187/Put/seqid=0 2024-11-15T16:58:14,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741909_1085 (size=13119) 2024-11-15T16:58:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T16:58:14,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741910_1086 (size=13119) 2024-11-15T16:58:14,513 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/06c8523e6f6c49b2830458e597e3fc29 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/06c8523e6f6c49b2830458e597e3fc29 2024-11-15T16:58:14,520 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into 06c8523e6f6c49b2830458e597e3fc29(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:14,520 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:14,520 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=12, startTime=1731689894471; duration=0sec 2024-11-15T16:58:14,521 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:14,521 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:14,521 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:14,522 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:14,523 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:14,523 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,523 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f3c2eaaf6ad04fcd9f51881676146e3e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/686e22e4778545aeb4385f12f0fd74b4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f2f966f32e07483ab47bfd607b994180, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/da8d4fd93f534329a627f048d36bbce1] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=48.7 K 2024-11-15T16:58:14,523 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c2eaaf6ad04fcd9f51881676146e3e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731689890878 2024-11-15T16:58:14,524 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 686e22e4778545aeb4385f12f0fd74b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731689891232 2024-11-15T16:58:14,524 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f966f32e07483ab47bfd607b994180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689891380 2024-11-15T16:58:14,524 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting da8d4fd93f534329a627f048d36bbce1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731689892040 2024-11-15T16:58:14,539 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:14,540 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f786066983ae437d8096102996478a93 is 50, key is test_row_0/C:col10/1731689893187/Put/seqid=0 2024-11-15T16:58:14,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741911_1087 (size=13119) 2024-11-15T16:58:14,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T16:58:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:14,603 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T16:58:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e32ce56b2f78446a95fde22a65ad4970 is 50, key is test_row_0/A:col10/1731689893200/Put/seqid=0 2024-11-15T16:58:14,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741912_1088 (size=12301) 2024-11-15T16:58:14,913 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/cbeef73413b64174bf261bbb0d593284 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/cbeef73413b64174bf261bbb0d593284 2024-11-15T16:58:14,920 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into cbeef73413b64174bf261bbb0d593284(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:14,920 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:14,920 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=12, startTime=1731689894471; duration=0sec 2024-11-15T16:58:14,924 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:14,924 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:14,955 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/f786066983ae437d8096102996478a93 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f786066983ae437d8096102996478a93 2024-11-15T16:58:14,962 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into f786066983ae437d8096102996478a93(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:14,962 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:14,962 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=12, startTime=1731689894471; duration=0sec 2024-11-15T16:58:14,962 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:14,962 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:15,021 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e32ce56b2f78446a95fde22a65ad4970 2024-11-15T16:58:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/609c847829e74a84a3d16e20410df73f is 50, key is test_row_0/B:col10/1731689893200/Put/seqid=0 2024-11-15T16:58:15,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741913_1089 (size=12301) 2024-11-15T16:58:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:15,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689955402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689955403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689955407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689955410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689955411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,438 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/609c847829e74a84a3d16e20410df73f 2024-11-15T16:58:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/507ba863dd4c417288faab08e66858e9 is 50, key is test_row_0/C:col10/1731689893200/Put/seqid=0 2024-11-15T16:58:15,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741914_1090 (size=12301) 2024-11-15T16:58:15,455 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/507ba863dd4c417288faab08e66858e9 2024-11-15T16:58:15,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e32ce56b2f78446a95fde22a65ad4970 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e32ce56b2f78446a95fde22a65ad4970 2024-11-15T16:58:15,466 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e32ce56b2f78446a95fde22a65ad4970, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T16:58:15,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/609c847829e74a84a3d16e20410df73f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/609c847829e74a84a3d16e20410df73f 2024-11-15T16:58:15,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/609c847829e74a84a3d16e20410df73f, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T16:58:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/507ba863dd4c417288faab08e66858e9 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/507ba863dd4c417288faab08e66858e9 2024-11-15T16:58:15,487 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/507ba863dd4c417288faab08e66858e9, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T16:58:15,488 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1195534959da145ee3aa98da7f864a98 in 885ms, sequenceid=370, compaction requested=false 2024-11-15T16:58:15,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:15,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:15,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T16:58:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T16:58:15,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T16:58:15,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-11-15T16:58:15,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1280 sec 2024-11-15T16:58:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T16:58:15,517 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:15,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:15,519 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:15,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T16:58:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-15T16:58:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:15,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T16:58:15,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:15,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:15,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/1d217a89f3ca4260a89729c3a1027704 is 50, key is test_row_0/A:col10/1731689895517/Put/seqid=0 2024-11-15T16:58:15,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689955530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689955532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689955533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689955536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689955536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741915_1091 (size=12301) 2024-11-15T16:58:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T16:58:15,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689955646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689955646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689955646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689955647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689955647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T16:58:15,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:15,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:15,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:15,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T16:58:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:15,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T16:58:15,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689955855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689955855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689955856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689955856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689955856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:15,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/1d217a89f3ca4260a89729c3a1027704 2024-11-15T16:58:15,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/84d2672a98ad4042840e0fe115983a37 is 50, key is test_row_0/B:col10/1731689895517/Put/seqid=0 2024-11-15T16:58:15,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741916_1092 (size=12301) 2024-11-15T16:58:15,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T16:58:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:15,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T16:58:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:16,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T16:58:16,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689956165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689956166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689956166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689956166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689956166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T16:58:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:16,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/84d2672a98ad4042840e0fe115983a37 2024-11-15T16:58:16,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/54d5176862c04a7d8ba589f7c26e9280 is 50, key is test_row_0/C:col10/1731689895517/Put/seqid=0 2024-11-15T16:58:16,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741917_1093 (size=12301) 2024-11-15T16:58:16,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/54d5176862c04a7d8ba589f7c26e9280 2024-11-15T16:58:16,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/1d217a89f3ca4260a89729c3a1027704 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1d217a89f3ca4260a89729c3a1027704 2024-11-15T16:58:16,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1d217a89f3ca4260a89729c3a1027704, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T16:58:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/84d2672a98ad4042840e0fe115983a37 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/84d2672a98ad4042840e0fe115983a37 2024-11-15T16:58:16,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/84d2672a98ad4042840e0fe115983a37, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T16:58:16,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/54d5176862c04a7d8ba589f7c26e9280 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5176862c04a7d8ba589f7c26e9280 2024-11-15T16:58:16,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5176862c04a7d8ba589f7c26e9280, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T16:58:16,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1195534959da145ee3aa98da7f864a98 in 891ms, sequenceid=398, compaction requested=true 2024-11-15T16:58:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:16,410 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:16,410 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:16,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:16,412 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:16,412 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:16,413 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,413 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/06c8523e6f6c49b2830458e597e3fc29, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/609c847829e74a84a3d16e20410df73f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/84d2672a98ad4042840e0fe115983a37] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.8 K 2024-11-15T16:58:16,413 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:16,413 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:16,413 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,413 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/cbeef73413b64174bf261bbb0d593284, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e32ce56b2f78446a95fde22a65ad4970, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1d217a89f3ca4260a89729c3a1027704] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.8 K 2024-11-15T16:58:16,414 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c8523e6f6c49b2830458e597e3fc29, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731689892040 2024-11-15T16:58:16,415 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbeef73413b64174bf261bbb0d593284, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731689892040 2024-11-15T16:58:16,415 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 609c847829e74a84a3d16e20410df73f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689893199 2024-11-15T16:58:16,415 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e32ce56b2f78446a95fde22a65ad4970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689893199 2024-11-15T16:58:16,416 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d2672a98ad4042840e0fe115983a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731689895517 2024-11-15T16:58:16,416 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d217a89f3ca4260a89729c3a1027704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731689895517 2024-11-15T16:58:16,430 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:16,431 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/d528ce78d49c458e9750518d8df9449a is 50, key is test_row_0/A:col10/1731689895517/Put/seqid=0 2024-11-15T16:58:16,432 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:16,433 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/2f76b5a78e8f451d9cd863f60349552f is 50, key is test_row_0/B:col10/1731689895517/Put/seqid=0 2024-11-15T16:58:16,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741918_1094 (size=13221) 2024-11-15T16:58:16,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T16:58:16,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,448 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T16:58:16,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741919_1095 (size=13221) 2024-11-15T16:58:16,457 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/d528ce78d49c458e9750518d8df9449a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d528ce78d49c458e9750518d8df9449a 2024-11-15T16:58:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/0cc7544190c8444ab5a425882cfaf01b is 50, key is test_row_1/A:col10/1731689895531/Put/seqid=0 2024-11-15T16:58:16,466 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/2f76b5a78e8f451d9cd863f60349552f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2f76b5a78e8f451d9cd863f60349552f 2024-11-15T16:58:16,467 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into d528ce78d49c458e9750518d8df9449a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:16,467 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:16,467 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=13, startTime=1731689896410; duration=0sec 2024-11-15T16:58:16,467 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:16,468 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:16,468 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:16,473 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:16,473 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:16,473 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,473 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f786066983ae437d8096102996478a93, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/507ba863dd4c417288faab08e66858e9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5176862c04a7d8ba589f7c26e9280] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=36.8 K 2024-11-15T16:58:16,474 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting f786066983ae437d8096102996478a93, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731689892040 2024-11-15T16:58:16,474 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into 2f76b5a78e8f451d9cd863f60349552f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:16,474 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:16,474 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=13, startTime=1731689896410; duration=0sec 2024-11-15T16:58:16,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:16,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:16,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741920_1096 (size=9857) 2024-11-15T16:58:16,475 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 507ba863dd4c417288faab08e66858e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689893199 2024-11-15T16:58:16,476 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54d5176862c04a7d8ba589f7c26e9280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731689895517 2024-11-15T16:58:16,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/0cc7544190c8444ab5a425882cfaf01b 2024-11-15T16:58:16,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/ed23c7b96a6744a3832e2d14842f7a8a is 50, key is test_row_1/B:col10/1731689895531/Put/seqid=0 2024-11-15T16:58:16,490 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:16,491 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/fdd584ee660a4044a8261e1bd1430b1b is 50, key is test_row_0/C:col10/1731689895517/Put/seqid=0 2024-11-15T16:58:16,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741921_1097 (size=9857) 2024-11-15T16:58:16,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/ed23c7b96a6744a3832e2d14842f7a8a 2024-11-15T16:58:16,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741922_1098 (size=13221) 2024-11-15T16:58:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/c562e60bbb7f433b930063e6ce654ced is 50, key is test_row_1/C:col10/1731689895531/Put/seqid=0 2024-11-15T16:58:16,518 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/fdd584ee660a4044a8261e1bd1430b1b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/fdd584ee660a4044a8261e1bd1430b1b 2024-11-15T16:58:16,525 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into fdd584ee660a4044a8261e1bd1430b1b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:16,525 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:16,525 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=13, startTime=1731689896410; duration=0sec 2024-11-15T16:58:16,526 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:16,526 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:16,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741923_1099 (size=9857) 2024-11-15T16:58:16,530 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/c562e60bbb7f433b930063e6ce654ced 2024-11-15T16:58:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/0cc7544190c8444ab5a425882cfaf01b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0cc7544190c8444ab5a425882cfaf01b 2024-11-15T16:58:16,543 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0cc7544190c8444ab5a425882cfaf01b, entries=100, sequenceid=408, filesize=9.6 K 2024-11-15T16:58:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/ed23c7b96a6744a3832e2d14842f7a8a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ed23c7b96a6744a3832e2d14842f7a8a 2024-11-15T16:58:16,554 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ed23c7b96a6744a3832e2d14842f7a8a, entries=100, sequenceid=408, filesize=9.6 K 2024-11-15T16:58:16,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/c562e60bbb7f433b930063e6ce654ced as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c562e60bbb7f433b930063e6ce654ced 2024-11-15T16:58:16,561 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c562e60bbb7f433b930063e6ce654ced, entries=100, sequenceid=408, filesize=9.6 K 2024-11-15T16:58:16,562 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 1195534959da145ee3aa98da7f864a98 in 115ms, sequenceid=408, compaction requested=false 2024-11-15T16:58:16,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-15T16:58:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-15T16:58:16,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T16:58:16,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0410 sec 2024-11-15T16:58:16,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.0490 sec 2024-11-15T16:58:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T16:58:16,656 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:16,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-15T16:58:16,660 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T16:58:16,661 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:16,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:16,688 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:16,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T16:58:16,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9cfeb542608241fbb143ad5e64faf1b6 is 50, key is test_row_0/A:col10/1731689896685/Put/seqid=0 2024-11-15T16:58:16,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741924_1100 (size=12301) 2024-11-15T16:58:16,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689956723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689956725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689956727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689956729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689956730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T16:58:16,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689956836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689956836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689956836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689956846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689956846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:16,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:16,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T16:58:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689957046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689957046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689957046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689957055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689957056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9cfeb542608241fbb143ad5e64faf1b6 2024-11-15T16:58:17,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/987fba8d794a4652ae8b6fdd444cbf4b is 50, key is test_row_0/B:col10/1731689896685/Put/seqid=0 2024-11-15T16:58:17,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741925_1101 (size=12301) 2024-11-15T16:58:17,273 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T16:58:17,274 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:17,274 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T16:58:17,274 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:17,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,279 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T16:58:17,279 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,280 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,280 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,281 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,281 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,281 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:17,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:17,283 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T16:58:17,283 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:17,283 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,283 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:17,283 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:17,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T16:58:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689957356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689957355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689957355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689957366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689957366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/987fba8d794a4652ae8b6fdd444cbf4b 2024-11-15T16:58:17,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/54d5d3f1f0084b129aecf5a13d5a9e0d is 50, key is test_row_0/C:col10/1731689896685/Put/seqid=0 2024-11-15T16:58:17,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741926_1102 (size=12301) 2024-11-15T16:58:17,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:17,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T16:58:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1731689957866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1731689957866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53278 deadline: 1731689957866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53246 deadline: 1731689957876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731689957876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:17,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:17,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:17,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:17,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:17,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/54d5d3f1f0084b129aecf5a13d5a9e0d 2024-11-15T16:58:17,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/9cfeb542608241fbb143ad5e64faf1b6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9cfeb542608241fbb143ad5e64faf1b6 2024-11-15T16:58:17,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9cfeb542608241fbb143ad5e64faf1b6, entries=150, sequenceid=423, filesize=12.0 K 2024-11-15T16:58:17,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/987fba8d794a4652ae8b6fdd444cbf4b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/987fba8d794a4652ae8b6fdd444cbf4b 2024-11-15T16:58:17,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/987fba8d794a4652ae8b6fdd444cbf4b, entries=150, sequenceid=423, filesize=12.0 K 2024-11-15T16:58:17,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/54d5d3f1f0084b129aecf5a13d5a9e0d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5d3f1f0084b129aecf5a13d5a9e0d 2024-11-15T16:58:18,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5d3f1f0084b129aecf5a13d5a9e0d, entries=150, sequenceid=423, filesize=12.0 K 2024-11-15T16:58:18,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1195534959da145ee3aa98da7f864a98 in 1314ms, sequenceid=423, compaction requested=true 2024-11-15T16:58:18,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:18,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:18,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:18,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:18,002 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:18,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:18,003 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1195534959da145ee3aa98da7f864a98:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:18,003 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:18,003 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:18,003 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/A is initiating minor compaction (all files) 2024-11-15T16:58:18,004 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/B is initiating minor compaction (all files) 2024-11-15T16:58:18,004 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/A in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:18,004 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/B in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:18,004 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d528ce78d49c458e9750518d8df9449a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0cc7544190c8444ab5a425882cfaf01b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9cfeb542608241fbb143ad5e64faf1b6] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=34.5 K 2024-11-15T16:58:18,004 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2f76b5a78e8f451d9cd863f60349552f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ed23c7b96a6744a3832e2d14842f7a8a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/987fba8d794a4652ae8b6fdd444cbf4b] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=34.5 K 2024-11-15T16:58:18,004 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d528ce78d49c458e9750518d8df9449a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731689895517 2024-11-15T16:58:18,004 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f76b5a78e8f451d9cd863f60349552f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731689895517 2024-11-15T16:58:18,004 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cc7544190c8444ab5a425882cfaf01b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731689895531 2024-11-15T16:58:18,005 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting ed23c7b96a6744a3832e2d14842f7a8a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731689895531 2024-11-15T16:58:18,005 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cfeb542608241fbb143ad5e64faf1b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731689896685 2024-11-15T16:58:18,005 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 987fba8d794a4652ae8b6fdd444cbf4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731689896685 2024-11-15T16:58:18,012 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#B#compaction#90 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:18,012 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#A#compaction#91 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:18,013 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/13ef9bee7f4a4d149d82961b974d77f8 is 50, key is test_row_0/B:col10/1731689896685/Put/seqid=0 2024-11-15T16:58:18,013 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e6d042243eb446f9aa6606f022bec0ed is 50, key is test_row_0/A:col10/1731689896685/Put/seqid=0 2024-11-15T16:58:18,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741927_1103 (size=13323) 2024-11-15T16:58:18,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741928_1104 (size=13323) 2024-11-15T16:58:18,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T16:58:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:18,051 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T16:58:18,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/12f8d9f533864c738afb0da44a12dc5a is 50, key is test_row_0/A:col10/1731689896722/Put/seqid=0 2024-11-15T16:58:18,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741929_1105 (size=12301) 2024-11-15T16:58:18,423 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/e6d042243eb446f9aa6606f022bec0ed as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e6d042243eb446f9aa6606f022bec0ed 2024-11-15T16:58:18,424 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/13ef9bee7f4a4d149d82961b974d77f8 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/13ef9bee7f4a4d149d82961b974d77f8 2024-11-15T16:58:18,429 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/A of 1195534959da145ee3aa98da7f864a98 into e6d042243eb446f9aa6606f022bec0ed(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:18,429 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:18,430 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/A, priority=13, startTime=1731689898002; duration=0sec 2024-11-15T16:58:18,430 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:18,430 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:A 2024-11-15T16:58:18,430 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:18,430 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/B of 1195534959da145ee3aa98da7f864a98 into 13ef9bee7f4a4d149d82961b974d77f8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:18,430 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:18,430 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/B, priority=13, startTime=1731689898002; duration=0sec 2024-11-15T16:58:18,430 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:18,430 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:B 2024-11-15T16:58:18,431 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:18,431 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 1195534959da145ee3aa98da7f864a98/C is initiating minor compaction (all files) 2024-11-15T16:58:18,431 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1195534959da145ee3aa98da7f864a98/C in TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:18,431 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/fdd584ee660a4044a8261e1bd1430b1b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c562e60bbb7f433b930063e6ce654ced, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5d3f1f0084b129aecf5a13d5a9e0d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp, totalSize=34.5 K 2024-11-15T16:58:18,432 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdd584ee660a4044a8261e1bd1430b1b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731689895517 2024-11-15T16:58:18,432 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c562e60bbb7f433b930063e6ce654ced, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731689895531 2024-11-15T16:58:18,432 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54d5d3f1f0084b129aecf5a13d5a9e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731689896685 2024-11-15T16:58:18,440 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1195534959da145ee3aa98da7f864a98#C#compaction#93 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:18,441 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/17a1b20f1662439db3a0058ed705b709 is 50, key is test_row_0/C:col10/1731689896685/Put/seqid=0 2024-11-15T16:58:18,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741930_1106 (size=13323) 2024-11-15T16:58:18,461 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/12f8d9f533864c738afb0da44a12dc5a 2024-11-15T16:58:18,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0721b43bd99f40bb8162d425e58bf197 is 50, key is test_row_0/B:col10/1731689896722/Put/seqid=0 2024-11-15T16:58:18,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741931_1107 (size=12301) 2024-11-15T16:58:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T16:58:18,851 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/17a1b20f1662439db3a0058ed705b709 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/17a1b20f1662439db3a0058ed705b709 2024-11-15T16:58:18,857 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1195534959da145ee3aa98da7f864a98/C of 1195534959da145ee3aa98da7f864a98 into 17a1b20f1662439db3a0058ed705b709(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:18,857 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:18,857 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., storeName=1195534959da145ee3aa98da7f864a98/C, priority=13, startTime=1731689898003; duration=0sec 2024-11-15T16:58:18,857 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:18,857 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1195534959da145ee3aa98da7f864a98:C 2024-11-15T16:58:18,873 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0721b43bd99f40bb8162d425e58bf197 2024-11-15T16:58:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. as already flushing 2024-11-15T16:58:18,877 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T16:58:18,877 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:18,877 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T16:58:18,877 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,877 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:18,877 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,877 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:18,877 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/87ecc605a0124d1085a26b5202d6fb2d is 50, key is test_row_0/C:col10/1731689896722/Put/seqid=0 2024-11-15T16:58:18,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741932_1108 (size=12301) 2024-11-15T16:58:18,887 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T16:58:18,887 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:18,887 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,888 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T16:58:18,888 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T16:58:18,888 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:18,888 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:18,888 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,888 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:18,888 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,888 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:18,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:18,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:18,888 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:19,285 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/87ecc605a0124d1085a26b5202d6fb2d 2024-11-15T16:58:19,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/12f8d9f533864c738afb0da44a12dc5a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/12f8d9f533864c738afb0da44a12dc5a 2024-11-15T16:58:19,297 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/12f8d9f533864c738afb0da44a12dc5a, entries=150, sequenceid=447, filesize=12.0 K 2024-11-15T16:58:19,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/0721b43bd99f40bb8162d425e58bf197 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0721b43bd99f40bb8162d425e58bf197 2024-11-15T16:58:19,303 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0721b43bd99f40bb8162d425e58bf197, entries=150, sequenceid=447, filesize=12.0 K 2024-11-15T16:58:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/87ecc605a0124d1085a26b5202d6fb2d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/87ecc605a0124d1085a26b5202d6fb2d 2024-11-15T16:58:19,308 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/87ecc605a0124d1085a26b5202d6fb2d, entries=150, sequenceid=447, filesize=12.0 K 2024-11-15T16:58:19,309 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=33.54 KB/34350 for 1195534959da145ee3aa98da7f864a98 in 1259ms, sequenceid=447, compaction requested=false 2024-11-15T16:58:19,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 1195534959da145ee3aa98da7f864a98: 2024-11-15T16:58:19,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:19,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-15T16:58:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-15T16:58:19,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T16:58:19,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6500 sec 2024-11-15T16:58:19,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.6570 sec 2024-11-15T16:58:20,498 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T16:58:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T16:58:20,827 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3715 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3609 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1625 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4871 rows 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1628 2024-11-15T16:58:20,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4880 rows 2024-11-15T16:58:20,827 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:58:20,828 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:58:20,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:20,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:20,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:20,832 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T16:58:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T16:58:20,842 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689900842"}]},"ts":"1731689900842"} 2024-11-15T16:58:20,844 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T16:58:20,844 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T16:58:20,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T16:58:20,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, UNASSIGN}] 2024-11-15T16:58:20,852 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, UNASSIGN 2024-11-15T16:58:20,854 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1195534959da145ee3aa98da7f864a98, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:20,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, UNASSIGN because future has completed 2024-11-15T16:58:20,866 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:58:20,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:58:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T16:58:21,026 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:21,026 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:58:21,027 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 1195534959da145ee3aa98da7f864a98, disabling compactions & flushes 2024-11-15T16:58:21,027 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:21,027 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:21,027 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. after waiting 0 ms 2024-11-15T16:58:21,027 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:21,027 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 1195534959da145ee3aa98da7f864a98 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T16:58:21,033 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a6af31ebefe94c458862da67b6c1a8d6 is 50, key is test_row_1/A:col10/1731689898886/Put/seqid=0 2024-11-15T16:58:21,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741933_1109 (size=9857) 2024-11-15T16:58:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T16:58:21,442 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a6af31ebefe94c458862da67b6c1a8d6 2024-11-15T16:58:21,454 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/6b22cfd7e7af4eeaac646e031d191e6a is 50, key is test_row_1/B:col10/1731689898886/Put/seqid=0 2024-11-15T16:58:21,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741934_1110 (size=9857) 2024-11-15T16:58:21,460 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/6b22cfd7e7af4eeaac646e031d191e6a 2024-11-15T16:58:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T16:58:21,472 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/1562f7e755564e8f930e0f5007283f95 is 50, key is test_row_1/C:col10/1731689898886/Put/seqid=0 2024-11-15T16:58:21,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741935_1111 (size=9857) 2024-11-15T16:58:21,878 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/1562f7e755564e8f930e0f5007283f95 2024-11-15T16:58:21,886 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/A/a6af31ebefe94c458862da67b6c1a8d6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a6af31ebefe94c458862da67b6c1a8d6 2024-11-15T16:58:21,892 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a6af31ebefe94c458862da67b6c1a8d6, entries=100, sequenceid=458, filesize=9.6 K 2024-11-15T16:58:21,893 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/B/6b22cfd7e7af4eeaac646e031d191e6a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/6b22cfd7e7af4eeaac646e031d191e6a 2024-11-15T16:58:21,898 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/6b22cfd7e7af4eeaac646e031d191e6a, entries=100, sequenceid=458, filesize=9.6 K 2024-11-15T16:58:21,899 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/.tmp/C/1562f7e755564e8f930e0f5007283f95 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1562f7e755564e8f930e0f5007283f95 2024-11-15T16:58:21,903 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1562f7e755564e8f930e0f5007283f95, entries=100, sequenceid=458, filesize=9.6 K 2024-11-15T16:58:21,905 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1195534959da145ee3aa98da7f864a98 in 877ms, sequenceid=458, compaction requested=true 2024-11-15T16:58:21,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/ddc360dff76f4d6ba8fdf2f88cd122b0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e2e0d076df3248d78aff35a0e33eb8c7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a7492c2f05ef4f1a941e96c328c99c68, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0dd242fc74a44ff08c7650c7cafb7172, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1c91fa4f5a6a479487069e8ef7be1c07, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e8c669eaf860451696ee5253cfba75e3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e76d47b9b74649f7a78ba8cb5968f58f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d53df0d3d184413abef30f92b1e7900e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5902f0ef44994e74950668b571006ab1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/6b2a138f36894f26a49d288d5cc412ff, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5b5a1b317c8e438087a0ce1d036f9fdb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3f602e7dbcd549f08702b9b264e4adea, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bf9f15478e0541f58370a37da894b9aa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bda3c4efd627452f82308d6843c287cf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/85fac4aacbd2481282ee6ee58f573484, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/39afb6ca361c4cbabd216a14b1267b69, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a25b0812584b4bc7ae82391beb9e2e20, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9acc169de2f543cf98e2c89bb9e0592f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3426c2ba610c4cbe90918542aa702505, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5ae0805e9c784185a40ee917afe92e19, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/b5d89e54221f49d997ec0c7040a97a96, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9e1bf367874e4cb8a140d869a272035a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9abef813e3ef4110ba2059f8d64141e0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/cbeef73413b64174bf261bbb0d593284, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/8a8ae3c2e649406596c6e57f89c3067a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e32ce56b2f78446a95fde22a65ad4970, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d528ce78d49c458e9750518d8df9449a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1d217a89f3ca4260a89729c3a1027704, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0cc7544190c8444ab5a425882cfaf01b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9cfeb542608241fbb143ad5e64faf1b6] to archive 2024-11-15T16:58:21,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:58:21,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/ddc360dff76f4d6ba8fdf2f88cd122b0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/ddc360dff76f4d6ba8fdf2f88cd122b0 2024-11-15T16:58:21,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e2e0d076df3248d78aff35a0e33eb8c7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e2e0d076df3248d78aff35a0e33eb8c7 2024-11-15T16:58:21,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a7492c2f05ef4f1a941e96c328c99c68 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a7492c2f05ef4f1a941e96c328c99c68 2024-11-15T16:58:21,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0dd242fc74a44ff08c7650c7cafb7172 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0dd242fc74a44ff08c7650c7cafb7172 2024-11-15T16:58:21,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1c91fa4f5a6a479487069e8ef7be1c07 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1c91fa4f5a6a479487069e8ef7be1c07 2024-11-15T16:58:21,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e8c669eaf860451696ee5253cfba75e3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e8c669eaf860451696ee5253cfba75e3 2024-11-15T16:58:21,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e76d47b9b74649f7a78ba8cb5968f58f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e76d47b9b74649f7a78ba8cb5968f58f 2024-11-15T16:58:21,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d53df0d3d184413abef30f92b1e7900e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d53df0d3d184413abef30f92b1e7900e 2024-11-15T16:58:21,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5902f0ef44994e74950668b571006ab1 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5902f0ef44994e74950668b571006ab1 2024-11-15T16:58:21,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/6b2a138f36894f26a49d288d5cc412ff to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/6b2a138f36894f26a49d288d5cc412ff 2024-11-15T16:58:21,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5b5a1b317c8e438087a0ce1d036f9fdb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5b5a1b317c8e438087a0ce1d036f9fdb 2024-11-15T16:58:21,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3f602e7dbcd549f08702b9b264e4adea to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3f602e7dbcd549f08702b9b264e4adea 2024-11-15T16:58:21,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bf9f15478e0541f58370a37da894b9aa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bf9f15478e0541f58370a37da894b9aa 2024-11-15T16:58:21,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bda3c4efd627452f82308d6843c287cf to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/bda3c4efd627452f82308d6843c287cf 2024-11-15T16:58:21,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/85fac4aacbd2481282ee6ee58f573484 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/85fac4aacbd2481282ee6ee58f573484 2024-11-15T16:58:21,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/39afb6ca361c4cbabd216a14b1267b69 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/39afb6ca361c4cbabd216a14b1267b69 2024-11-15T16:58:21,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a25b0812584b4bc7ae82391beb9e2e20 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a25b0812584b4bc7ae82391beb9e2e20 2024-11-15T16:58:21,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9acc169de2f543cf98e2c89bb9e0592f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9acc169de2f543cf98e2c89bb9e0592f 2024-11-15T16:58:21,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3426c2ba610c4cbe90918542aa702505 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/3426c2ba610c4cbe90918542aa702505 2024-11-15T16:58:21,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5ae0805e9c784185a40ee917afe92e19 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/5ae0805e9c784185a40ee917afe92e19 2024-11-15T16:58:21,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/b5d89e54221f49d997ec0c7040a97a96 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/b5d89e54221f49d997ec0c7040a97a96 2024-11-15T16:58:21,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9e1bf367874e4cb8a140d869a272035a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9e1bf367874e4cb8a140d869a272035a 2024-11-15T16:58:21,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9abef813e3ef4110ba2059f8d64141e0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9abef813e3ef4110ba2059f8d64141e0 2024-11-15T16:58:21,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/cbeef73413b64174bf261bbb0d593284 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/cbeef73413b64174bf261bbb0d593284 2024-11-15T16:58:21,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/8a8ae3c2e649406596c6e57f89c3067a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/8a8ae3c2e649406596c6e57f89c3067a 2024-11-15T16:58:21,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e32ce56b2f78446a95fde22a65ad4970 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e32ce56b2f78446a95fde22a65ad4970 2024-11-15T16:58:21,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d528ce78d49c458e9750518d8df9449a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/d528ce78d49c458e9750518d8df9449a 2024-11-15T16:58:21,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1d217a89f3ca4260a89729c3a1027704 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/1d217a89f3ca4260a89729c3a1027704 2024-11-15T16:58:21,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0cc7544190c8444ab5a425882cfaf01b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/0cc7544190c8444ab5a425882cfaf01b 2024-11-15T16:58:21,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9cfeb542608241fbb143ad5e64faf1b6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/9cfeb542608241fbb143ad5e64faf1b6 2024-11-15T16:58:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T16:58:21,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/80b991b7c23b4b69a9a909bdb3159b04, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/1964518c989b4daf8b23c1900acb21b8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e9b6ce2ad17149cf936c87494a7855c0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/3af7128b91c84c45ba1ef25e830f4189, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/20e5970bdd2342e99e5e76f7f52d4cf7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0c52d4d81d9a4735a0e3a41208ce72af, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0255b451b06341bca9ef9e7d7b53b418, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/f98d9a156dd64611be514cd44967ee48, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7d161bbcc5f64794977114f88d11caee, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/96fdbead822943ebb8ec3e79851ac9f6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e68eafd6b89d49c8a5cc0c8708b83baa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/93101a40b7634b7bb987049f033ac413, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/691cde167e0840b9a97347e01e34e07e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/15fb9f98e87f41538fb59eb9d07c8757, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2100e6b8c9134104a2f5ba0fee7bf686, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ef8acd56c5bd4cd3af0d7f2d339cec8a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/8fa00bebde584d6eb7758cd648c7cad2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7f9194bbac3a427686aa54f1f60f7d72, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/443b70d25aef4435af49aa21ec4f7f1e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/a46b89bc7ce441b6a65fb82da8eb7c3c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/4f4573886d324588929a014177a6d2b7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/16c60302d07345faadf12d4a12207f8c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/06c8523e6f6c49b2830458e597e3fc29, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/b340e54e30a24fea80c82f2bc692b4a1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/609c847829e74a84a3d16e20410df73f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2f76b5a78e8f451d9cd863f60349552f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/84d2672a98ad4042840e0fe115983a37, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ed23c7b96a6744a3832e2d14842f7a8a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/987fba8d794a4652ae8b6fdd444cbf4b] to archive 2024-11-15T16:58:21,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:58:21,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/80b991b7c23b4b69a9a909bdb3159b04 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/80b991b7c23b4b69a9a909bdb3159b04 2024-11-15T16:58:21,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e1082bfd822c4bf8bc0ef1e9e55b2ca6 2024-11-15T16:58:21,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/1964518c989b4daf8b23c1900acb21b8 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/1964518c989b4daf8b23c1900acb21b8 2024-11-15T16:58:21,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e9b6ce2ad17149cf936c87494a7855c0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e9b6ce2ad17149cf936c87494a7855c0 2024-11-15T16:58:21,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/3af7128b91c84c45ba1ef25e830f4189 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/3af7128b91c84c45ba1ef25e830f4189 2024-11-15T16:58:21,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/20e5970bdd2342e99e5e76f7f52d4cf7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/20e5970bdd2342e99e5e76f7f52d4cf7 2024-11-15T16:58:21,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0c52d4d81d9a4735a0e3a41208ce72af to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0c52d4d81d9a4735a0e3a41208ce72af 2024-11-15T16:58:21,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0255b451b06341bca9ef9e7d7b53b418 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0255b451b06341bca9ef9e7d7b53b418 2024-11-15T16:58:21,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/f98d9a156dd64611be514cd44967ee48 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/f98d9a156dd64611be514cd44967ee48 2024-11-15T16:58:22,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7d161bbcc5f64794977114f88d11caee to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7d161bbcc5f64794977114f88d11caee 2024-11-15T16:58:22,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/96fdbead822943ebb8ec3e79851ac9f6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/96fdbead822943ebb8ec3e79851ac9f6 2024-11-15T16:58:22,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e68eafd6b89d49c8a5cc0c8708b83baa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/e68eafd6b89d49c8a5cc0c8708b83baa 2024-11-15T16:58:22,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/93101a40b7634b7bb987049f033ac413 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/93101a40b7634b7bb987049f033ac413 2024-11-15T16:58:22,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/691cde167e0840b9a97347e01e34e07e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/691cde167e0840b9a97347e01e34e07e 2024-11-15T16:58:22,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/15fb9f98e87f41538fb59eb9d07c8757 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/15fb9f98e87f41538fb59eb9d07c8757 2024-11-15T16:58:22,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2100e6b8c9134104a2f5ba0fee7bf686 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2100e6b8c9134104a2f5ba0fee7bf686 2024-11-15T16:58:22,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ef8acd56c5bd4cd3af0d7f2d339cec8a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ef8acd56c5bd4cd3af0d7f2d339cec8a 2024-11-15T16:58:22,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/8fa00bebde584d6eb7758cd648c7cad2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/8fa00bebde584d6eb7758cd648c7cad2 2024-11-15T16:58:22,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7f9194bbac3a427686aa54f1f60f7d72 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/7f9194bbac3a427686aa54f1f60f7d72 2024-11-15T16:58:22,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/443b70d25aef4435af49aa21ec4f7f1e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/443b70d25aef4435af49aa21ec4f7f1e 2024-11-15T16:58:22,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/a46b89bc7ce441b6a65fb82da8eb7c3c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/a46b89bc7ce441b6a65fb82da8eb7c3c 2024-11-15T16:58:22,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/4f4573886d324588929a014177a6d2b7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/4f4573886d324588929a014177a6d2b7 2024-11-15T16:58:22,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/16c60302d07345faadf12d4a12207f8c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/16c60302d07345faadf12d4a12207f8c 2024-11-15T16:58:22,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/06c8523e6f6c49b2830458e597e3fc29 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/06c8523e6f6c49b2830458e597e3fc29 2024-11-15T16:58:22,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/b340e54e30a24fea80c82f2bc692b4a1 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/b340e54e30a24fea80c82f2bc692b4a1 2024-11-15T16:58:22,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/609c847829e74a84a3d16e20410df73f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/609c847829e74a84a3d16e20410df73f 2024-11-15T16:58:22,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2f76b5a78e8f451d9cd863f60349552f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/2f76b5a78e8f451d9cd863f60349552f 2024-11-15T16:58:22,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/84d2672a98ad4042840e0fe115983a37 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/84d2672a98ad4042840e0fe115983a37 2024-11-15T16:58:22,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ed23c7b96a6744a3832e2d14842f7a8a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/ed23c7b96a6744a3832e2d14842f7a8a 2024-11-15T16:58:22,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/987fba8d794a4652ae8b6fdd444cbf4b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/987fba8d794a4652ae8b6fdd444cbf4b 2024-11-15T16:58:22,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1c138084c36f4c10bdabb1d3e15e7552, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/403965bf889d491bbaf3e0172abe6dc0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/89992617cf94489a89e94a963821406e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/2235deaf87fa4f9b93ca0029dded7f6e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/0f1a0cc430d44ce6b18aa9e1871fc94d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ef68e567f8cc4aa8b36d7eb1407d80ff, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e3a2bda6efd7417ea831bff8a43d2ab6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/5b527b36b4624b5389b9ed759299cb80, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/167274f15db548629343ffb07a089132, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/55f9e2959602457480f6c76b71ebb9ba, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c753f95e038c47a88ec2c478fb53c4aa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/03d5f917a1ad43f5b1d92711b6b4f026, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ff14f00ec137411facc3c8678f75a854, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e27d54168437474eb2fafd54593b8da9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/4707f5446d2b4bf0baae5c59c047ec71, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/8e12bc8068f24725b863520a2c36a5c5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/9f37ebfeb6a8413b9f02d1a393e14742, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e7734291685243fca8d302573f107fe5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f3c2eaaf6ad04fcd9f51881676146e3e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e087b6c49ae84baeaad95864156443bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/686e22e4778545aeb4385f12f0fd74b4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f2f966f32e07483ab47bfd607b994180, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f786066983ae437d8096102996478a93, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/da8d4fd93f534329a627f048d36bbce1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/507ba863dd4c417288faab08e66858e9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/fdd584ee660a4044a8261e1bd1430b1b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5176862c04a7d8ba589f7c26e9280, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c562e60bbb7f433b930063e6ce654ced, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5d3f1f0084b129aecf5a13d5a9e0d] to archive 2024-11-15T16:58:22,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:58:22,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1c138084c36f4c10bdabb1d3e15e7552 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1c138084c36f4c10bdabb1d3e15e7552 2024-11-15T16:58:22,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/403965bf889d491bbaf3e0172abe6dc0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/403965bf889d491bbaf3e0172abe6dc0 2024-11-15T16:58:22,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/89992617cf94489a89e94a963821406e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/89992617cf94489a89e94a963821406e 2024-11-15T16:58:22,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/2235deaf87fa4f9b93ca0029dded7f6e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/2235deaf87fa4f9b93ca0029dded7f6e 2024-11-15T16:58:22,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/0f1a0cc430d44ce6b18aa9e1871fc94d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/0f1a0cc430d44ce6b18aa9e1871fc94d 2024-11-15T16:58:22,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ef68e567f8cc4aa8b36d7eb1407d80ff to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ef68e567f8cc4aa8b36d7eb1407d80ff 2024-11-15T16:58:22,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e3a2bda6efd7417ea831bff8a43d2ab6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e3a2bda6efd7417ea831bff8a43d2ab6 2024-11-15T16:58:22,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/5b527b36b4624b5389b9ed759299cb80 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/5b527b36b4624b5389b9ed759299cb80 2024-11-15T16:58:22,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f36e5bd5ef1a43e3a6d6d2fa34ca0d28 2024-11-15T16:58:22,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/167274f15db548629343ffb07a089132 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/167274f15db548629343ffb07a089132 2024-11-15T16:58:22,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/55f9e2959602457480f6c76b71ebb9ba to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/55f9e2959602457480f6c76b71ebb9ba 2024-11-15T16:58:22,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c753f95e038c47a88ec2c478fb53c4aa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c753f95e038c47a88ec2c478fb53c4aa 2024-11-15T16:58:22,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/03d5f917a1ad43f5b1d92711b6b4f026 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/03d5f917a1ad43f5b1d92711b6b4f026 2024-11-15T16:58:22,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ff14f00ec137411facc3c8678f75a854 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/ff14f00ec137411facc3c8678f75a854 2024-11-15T16:58:22,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e27d54168437474eb2fafd54593b8da9 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e27d54168437474eb2fafd54593b8da9 2024-11-15T16:58:22,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/4707f5446d2b4bf0baae5c59c047ec71 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/4707f5446d2b4bf0baae5c59c047ec71 2024-11-15T16:58:22,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/8e12bc8068f24725b863520a2c36a5c5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/8e12bc8068f24725b863520a2c36a5c5 2024-11-15T16:58:22,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/9f37ebfeb6a8413b9f02d1a393e14742 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/9f37ebfeb6a8413b9f02d1a393e14742 2024-11-15T16:58:22,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e7734291685243fca8d302573f107fe5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e7734291685243fca8d302573f107fe5 2024-11-15T16:58:22,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f3c2eaaf6ad04fcd9f51881676146e3e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f3c2eaaf6ad04fcd9f51881676146e3e 2024-11-15T16:58:22,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e087b6c49ae84baeaad95864156443bb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/e087b6c49ae84baeaad95864156443bb 2024-11-15T16:58:22,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/686e22e4778545aeb4385f12f0fd74b4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/686e22e4778545aeb4385f12f0fd74b4 2024-11-15T16:58:22,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f2f966f32e07483ab47bfd607b994180 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f2f966f32e07483ab47bfd607b994180 2024-11-15T16:58:22,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f786066983ae437d8096102996478a93 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/f786066983ae437d8096102996478a93 2024-11-15T16:58:22,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/da8d4fd93f534329a627f048d36bbce1 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/da8d4fd93f534329a627f048d36bbce1 2024-11-15T16:58:22,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/507ba863dd4c417288faab08e66858e9 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/507ba863dd4c417288faab08e66858e9 2024-11-15T16:58:22,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/fdd584ee660a4044a8261e1bd1430b1b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/fdd584ee660a4044a8261e1bd1430b1b 2024-11-15T16:58:22,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5176862c04a7d8ba589f7c26e9280 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5176862c04a7d8ba589f7c26e9280 2024-11-15T16:58:22,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c562e60bbb7f433b930063e6ce654ced to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/c562e60bbb7f433b930063e6ce654ced 2024-11-15T16:58:22,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5d3f1f0084b129aecf5a13d5a9e0d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/54d5d3f1f0084b129aecf5a13d5a9e0d 2024-11-15T16:58:22,070 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/recovered.edits/461.seqid, newMaxSeqId=461, maxSeqId=1 2024-11-15T16:58:22,071 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. 2024-11-15T16:58:22,071 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 1195534959da145ee3aa98da7f864a98: Waiting for close lock at 1731689901027Running coprocessor pre-close hooks at 1731689901027Disabling compacts and flushes for region at 1731689901027Disabling writes for close at 1731689901027Obtaining lock to block concurrent updates at 1731689901027Preparing flush snapshotting stores in 1195534959da145ee3aa98da7f864a98 at 1731689901027Finished memstore snapshotting TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731689901028 (+1 ms)Flushing stores of TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98. at 1731689901028Flushing 1195534959da145ee3aa98da7f864a98/A: creating writer at 1731689901028Flushing 1195534959da145ee3aa98da7f864a98/A: appending metadata at 1731689901032 (+4 ms)Flushing 1195534959da145ee3aa98da7f864a98/A: closing flushed file at 1731689901032Flushing 1195534959da145ee3aa98da7f864a98/B: creating writer at 1731689901449 (+417 ms)Flushing 1195534959da145ee3aa98da7f864a98/B: appending metadata at 1731689901453 (+4 ms)Flushing 1195534959da145ee3aa98da7f864a98/B: closing flushed file at 1731689901453Flushing 1195534959da145ee3aa98da7f864a98/C: creating writer at 1731689901468 (+15 ms)Flushing 1195534959da145ee3aa98da7f864a98/C: appending metadata at 1731689901471 (+3 ms)Flushing 1195534959da145ee3aa98da7f864a98/C: closing flushed file at 1731689901472 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8c453fc: reopening flushed file at 1731689901885 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@358337ab: reopening flushed file at 1731689901892 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f90878e: reopening flushed file at 1731689901898 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1195534959da145ee3aa98da7f864a98 in 877ms, sequenceid=458, compaction requested=true at 1731689901905 (+7 ms)Writing region close event to WAL at 1731689902066 (+161 ms)Running coprocessor post-close hooks at 1731689902070 (+4 ms)Closed at 1731689902071 (+1 ms) 2024-11-15T16:58:22,073 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:22,074 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1195534959da145ee3aa98da7f864a98, regionState=CLOSED 2024-11-15T16:58:22,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:58:22,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T16:58:22,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 1195534959da145ee3aa98da7f864a98, server=a9bde5b3a1b5,35119,1731689872090 in 1.2110 sec 2024-11-15T16:58:22,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T16:58:22,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1195534959da145ee3aa98da7f864a98, UNASSIGN in 1.2280 sec 2024-11-15T16:58:22,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-15T16:58:22,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2360 sec 2024-11-15T16:58:22,086 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689902085"}]},"ts":"1731689902085"} 2024-11-15T16:58:22,087 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T16:58:22,088 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T16:58:22,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-11-15T16:58:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T16:58:22,986 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:22,989 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T16:58:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:22,995 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T16:58:22,996 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:23,002 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:23,006 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/recovered.edits] 2024-11-15T16:58:23,009 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/12f8d9f533864c738afb0da44a12dc5a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/12f8d9f533864c738afb0da44a12dc5a 2024-11-15T16:58:23,011 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a6af31ebefe94c458862da67b6c1a8d6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/a6af31ebefe94c458862da67b6c1a8d6 2024-11-15T16:58:23,012 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e6d042243eb446f9aa6606f022bec0ed to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/A/e6d042243eb446f9aa6606f022bec0ed 2024-11-15T16:58:23,015 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0721b43bd99f40bb8162d425e58bf197 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/0721b43bd99f40bb8162d425e58bf197 2024-11-15T16:58:23,016 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/13ef9bee7f4a4d149d82961b974d77f8 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/13ef9bee7f4a4d149d82961b974d77f8 2024-11-15T16:58:23,018 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/6b22cfd7e7af4eeaac646e031d191e6a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/B/6b22cfd7e7af4eeaac646e031d191e6a 2024-11-15T16:58:23,021 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1562f7e755564e8f930e0f5007283f95 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/1562f7e755564e8f930e0f5007283f95 2024-11-15T16:58:23,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/17a1b20f1662439db3a0058ed705b709 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/17a1b20f1662439db3a0058ed705b709 2024-11-15T16:58:23,023 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/87ecc605a0124d1085a26b5202d6fb2d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/C/87ecc605a0124d1085a26b5202d6fb2d 2024-11-15T16:58:23,026 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/recovered.edits/461.seqid to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98/recovered.edits/461.seqid 2024-11-15T16:58:23,027 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/1195534959da145ee3aa98da7f864a98 2024-11-15T16:58:23,027 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T16:58:23,033 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:23,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T16:58:23,043 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T16:58:23,079 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T16:58:23,081 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:23,081 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T16:58:23,082 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731689903081"}]},"ts":"9223372036854775807"} 2024-11-15T16:58:23,086 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T16:58:23,086 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1195534959da145ee3aa98da7f864a98, NAME => 'TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T16:58:23,087 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T16:58:23,087 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731689903087"}]},"ts":"9223372036854775807"} 2024-11-15T16:58:23,090 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T16:58:23,091 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:23,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 101 msec 2024-11-15T16:58:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T16:58:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T16:58:23,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:23,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;a9bde5b3a1b5:35119-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1850901185_22 at /127.0.0.1:51834 [Waiting for operation #153] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1850901185_22 at /127.0.0.1:50868 [Waiting for operation #240] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-915587138_22 at /127.0.0.1:50998 [Waiting for operation #220] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 177) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5871 (was 5360) - AvailableMemoryMB LEAK? - 2024-11-15T16:58:23,137 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=5869 2024-11-15T16:58:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T16:58:23,139 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:58:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:23,142 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:58:23,143 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:23,143 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-15T16:58:23,144 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:58:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T16:58:23,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741936_1112 (size=959) 2024-11-15T16:58:23,154 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T16:58:23,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741937_1113 (size=53) 2024-11-15T16:58:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T16:58:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T16:58:23,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:58:23,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5800e906b79c2ba33b2668d41c684c23, disabling compactions & flushes 2024-11-15T16:58:23,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:23,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:23,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. after waiting 0 ms 2024-11-15T16:58:23,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:23,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:23,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5800e906b79c2ba33b2668d41c684c23: Waiting for close lock at 1731689903572Disabling compacts and flushes for region at 1731689903572Disabling writes for close at 1731689903572Writing region close event to WAL at 1731689903572Closed at 1731689903572 2024-11-15T16:58:23,574 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:58:23,574 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731689903574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689903574"}]},"ts":"1731689903574"} 2024-11-15T16:58:23,576 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:58:23,577 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:58:23,577 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689903577"}]},"ts":"1731689903577"} 2024-11-15T16:58:23,579 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T16:58:23,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, ASSIGN}] 2024-11-15T16:58:23,581 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, ASSIGN 2024-11-15T16:58:23,581 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=false 2024-11-15T16:58:23,733 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:23,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, ASSIGN because future has completed 2024-11-15T16:58:23,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:58:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T16:58:23,894 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:23,894 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:58:23,895 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,895 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:58:23,895 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,895 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,897 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,900 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5800e906b79c2ba33b2668d41c684c23 columnFamilyName A 2024-11-15T16:58:23,900 DEBUG [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:23,901 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(327): Store=5800e906b79c2ba33b2668d41c684c23/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:23,901 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,903 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5800e906b79c2ba33b2668d41c684c23 columnFamilyName B 2024-11-15T16:58:23,903 DEBUG [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:23,904 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(327): Store=5800e906b79c2ba33b2668d41c684c23/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:23,904 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,906 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5800e906b79c2ba33b2668d41c684c23 columnFamilyName C 2024-11-15T16:58:23,906 DEBUG [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:23,907 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(327): Store=5800e906b79c2ba33b2668d41c684c23/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:23,907 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,908 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,908 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,911 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,911 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,912 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:58:23,914 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,916 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:58:23,917 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5800e906b79c2ba33b2668d41c684c23; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64989509, jitterRate=-0.031580850481987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:58:23,917 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:23,918 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5800e906b79c2ba33b2668d41c684c23: Running coprocessor pre-open hook at 1731689903895Writing region info on filesystem at 1731689903895Initializing all the Stores at 1731689903896 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689903897 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689903897Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689903897Cleaning up temporary data from old regions at 1731689903911 (+14 ms)Running coprocessor post-open hooks at 1731689903917 (+6 ms)Region opened successfully at 1731689903918 (+1 ms) 2024-11-15T16:58:23,919 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., pid=34, masterSystemTime=1731689903889 2024-11-15T16:58:23,922 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:23,922 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:23,922 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:23,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:58:23,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T16:58:23,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 in 190 msec 2024-11-15T16:58:23,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T16:58:23,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, ASSIGN in 350 msec 2024-11-15T16:58:23,932 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:58:23,932 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689903932"}]},"ts":"1731689903932"} 2024-11-15T16:58:23,934 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T16:58:23,934 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:58:23,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 794 msec 2024-11-15T16:58:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T16:58:24,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:24,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,289 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63097a23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,292 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a870147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,298 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:58:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:58:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5445cd2b 2024-11-15T16:58:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:58:24,300 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:58:24,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T16:58:24,305 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:58:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:24,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741938_1114 (size=995) 2024-11-15T16:58:24,364 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T16:58:24,364 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T16:58:24,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T16:58:24,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, REOPEN/MOVE}] 2024-11-15T16:58:24,380 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, REOPEN/MOVE 2024-11-15T16:58:24,381 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:24,382 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41785 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a9bde5b3a1b5,35119,1731689872090, table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T16:58:24,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, REOPEN/MOVE because future has completed 2024-11-15T16:58:24,384 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:58:24,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:58:24,536 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,536 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:58:24,536 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 5800e906b79c2ba33b2668d41c684c23, disabling compactions & flushes 2024-11-15T16:58:24,537 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:24,537 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:24,537 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. after waiting 0 ms 2024-11-15T16:58:24,537 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:24,541 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T16:58:24,541 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:24,541 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 5800e906b79c2ba33b2668d41c684c23: Waiting for close lock at 1731689904536Running coprocessor pre-close hooks at 1731689904536Disabling compacts and flushes for region at 1731689904536Disabling writes for close at 1731689904537 (+1 ms)Writing region close event to WAL at 1731689904537Running coprocessor post-close hooks at 1731689904541 (+4 ms)Closed at 1731689904541 2024-11-15T16:58:24,542 WARN [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 5800e906b79c2ba33b2668d41c684c23 to self. 2024-11-15T16:58:24,544 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,544 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=CLOSED 2024-11-15T16:58:24,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:58:24,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-15T16:58:24,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 in 163 msec 2024-11-15T16:58:24,549 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, REOPEN/MOVE; state=CLOSED, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=true 2024-11-15T16:58:24,700 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:24,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, REOPEN/MOVE because future has completed 2024-11-15T16:58:24,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:58:24,858 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:24,858 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:58:24,858 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,858 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:58:24,858 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,858 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,863 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,868 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5800e906b79c2ba33b2668d41c684c23 columnFamilyName A 2024-11-15T16:58:24,870 DEBUG [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:24,870 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(327): Store=5800e906b79c2ba33b2668d41c684c23/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:24,871 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,872 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5800e906b79c2ba33b2668d41c684c23 columnFamilyName B 2024-11-15T16:58:24,872 DEBUG [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:24,872 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(327): Store=5800e906b79c2ba33b2668d41c684c23/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:24,872 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,873 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5800e906b79c2ba33b2668d41c684c23 columnFamilyName C 2024-11-15T16:58:24,873 DEBUG [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:24,873 INFO [StoreOpener-5800e906b79c2ba33b2668d41c684c23-1 {}] regionserver.HStore(327): Store=5800e906b79c2ba33b2668d41c684c23/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:24,873 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,874 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,875 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,876 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,876 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,876 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:58:24,877 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,878 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5800e906b79c2ba33b2668d41c684c23; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59907263, jitterRate=-0.10731221735477448}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:58:24,878 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:24,879 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5800e906b79c2ba33b2668d41c684c23: Running coprocessor pre-open hook at 1731689904858Writing region info on filesystem at 1731689904858Initializing all the Stores at 1731689904859 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689904859Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689904859Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689904859Cleaning up temporary data from old regions at 1731689904876 (+17 ms)Running coprocessor post-open hooks at 1731689904878 (+2 ms)Region opened successfully at 1731689904879 (+1 ms) 2024-11-15T16:58:24,880 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., pid=39, masterSystemTime=1731689904854 2024-11-15T16:58:24,882 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:24,882 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:24,883 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=OPEN, openSeqNum=5, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:24,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:58:24,887 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-15T16:58:24,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 in 184 msec 2024-11-15T16:58:24,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T16:58:24,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, REOPEN/MOVE in 510 msec 2024-11-15T16:58:24,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-15T16:58:24,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-15T16:58:24,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 587 msec 2024-11-15T16:58:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-15T16:58:24,899 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:24,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7309eae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9753ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,905 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678171c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,906 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@568a9f8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,909 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6fdcd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba4cd98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23eeb231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366f4416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c72161, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134f4a9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,921 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbefb49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,922 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f25e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,925 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64877518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3174ac96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,929 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b794f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,930 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4fb41a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:24,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:24,933 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399421b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:24,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:24,935 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@590288b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:24,936 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,945 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,946 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:24,948 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,949 DEBUG [Thread-596 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T16:58:24,950 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T16:58:24,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,951 DEBUG [Thread-596 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,951 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:24,952 DEBUG [Thread-598 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,954 DEBUG [Thread-598 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,954 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:24,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,956 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,957 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,965 DEBUG [Thread-591 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,965 DEBUG [Thread-591 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,966 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,967 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:24,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:58:24,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:24,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T16:58:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T16:58:25,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155da7b470cb20463f93a33334d4383507_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689904977/Put/seqid=0 2024-11-15T16:58:25,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689965052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689965053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689965054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689965057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689965067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741939_1115 (size=12154) 2024-11-15T16:58:25,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:25,098 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155da7b470cb20463f93a33334d4383507_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155da7b470cb20463f93a33334d4383507_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:25,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f9bb545c5eaf4424998696302510a0c8, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:25,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:25,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f9bb545c5eaf4424998696302510a0c8 is 175, key is test_row_0/A:col10/1731689904977/Put/seqid=0 2024-11-15T16:58:25,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741940_1116 (size=30955) 2024-11-15T16:58:25,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f9bb545c5eaf4424998696302510a0c8 2024-11-15T16:58:25,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/12672d8f888b4ce5924e4e5827532482 is 50, key is test_row_0/B:col10/1731689904977/Put/seqid=0 2024-11-15T16:58:25,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689965176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741941_1117 (size=12001) 2024-11-15T16:58:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689965178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/12672d8f888b4ce5924e4e5827532482 2024-11-15T16:58:25,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689965178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689965179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689965186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/48b1413cf92740788a66eeb4ca621179 is 50, key is test_row_0/C:col10/1731689904977/Put/seqid=0 2024-11-15T16:58:25,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741942_1118 (size=12001) 2024-11-15T16:58:25,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/48b1413cf92740788a66eeb4ca621179 2024-11-15T16:58:25,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f9bb545c5eaf4424998696302510a0c8 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f9bb545c5eaf4424998696302510a0c8 2024-11-15T16:58:25,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:25,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:25,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T16:58:25,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f9bb545c5eaf4424998696302510a0c8, entries=150, sequenceid=16, filesize=30.2 K 2024-11-15T16:58:25,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/12672d8f888b4ce5924e4e5827532482 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/12672d8f888b4ce5924e4e5827532482 2024-11-15T16:58:25,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/12672d8f888b4ce5924e4e5827532482, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T16:58:25,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/48b1413cf92740788a66eeb4ca621179 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48b1413cf92740788a66eeb4ca621179 2024-11-15T16:58:25,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48b1413cf92740788a66eeb4ca621179, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T16:58:25,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5800e906b79c2ba33b2668d41c684c23 in 315ms, sequenceid=16, compaction requested=false 2024-11-15T16:58:25,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:25,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:58:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:25,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115767c1134863d4d2fbd1b47767990c600_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689905041/Put/seqid=0 2024-11-15T16:58:25,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689965417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689965420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689965424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689965436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689965439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741943_1119 (size=14594) 2024-11-15T16:58:25,487 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:25,494 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115767c1134863d4d2fbd1b47767990c600_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115767c1134863d4d2fbd1b47767990c600_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:25,496 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d421549dd346463490f1fb85a3c27c3e, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:25,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d421549dd346463490f1fb85a3c27c3e is 175, key is test_row_0/A:col10/1731689905041/Put/seqid=0 2024-11-15T16:58:25,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741944_1120 (size=39549) 2024-11-15T16:58:25,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d421549dd346463490f1fb85a3c27c3e 2024-11-15T16:58:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689965546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689965546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689965547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/1566f598efb64bbcbc96d54b6390bf27 is 50, key is test_row_0/B:col10/1731689905041/Put/seqid=0 2024-11-15T16:58:25,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T16:58:25,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689965566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689965566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741945_1121 (size=12001) 2024-11-15T16:58:25,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689965766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689965766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689965767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689965787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689965787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:25,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:25,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:25,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:25,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:26,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/1566f598efb64bbcbc96d54b6390bf27 2024-11-15T16:58:26,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/e2186002e497421a9a40c1f9734c6f7a is 50, key is test_row_0/C:col10/1731689905041/Put/seqid=0 2024-11-15T16:58:26,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:26,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:26,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:26,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741946_1122 (size=12001) 2024-11-15T16:58:26,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/e2186002e497421a9a40c1f9734c6f7a 2024-11-15T16:58:26,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d421549dd346463490f1fb85a3c27c3e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d421549dd346463490f1fb85a3c27c3e 2024-11-15T16:58:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689966075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689966076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689966076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d421549dd346463490f1fb85a3c27c3e, entries=200, sequenceid=41, filesize=38.6 K 2024-11-15T16:58:26,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/1566f598efb64bbcbc96d54b6390bf27 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1566f598efb64bbcbc96d54b6390bf27 2024-11-15T16:58:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T16:58:26,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689966095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689966096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1566f598efb64bbcbc96d54b6390bf27, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T16:58:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/e2186002e497421a9a40c1f9734c6f7a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/e2186002e497421a9a40c1f9734c6f7a 2024-11-15T16:58:26,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/e2186002e497421a9a40c1f9734c6f7a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T16:58:26,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5800e906b79c2ba33b2668d41c684c23 in 709ms, sequenceid=41, compaction requested=false 2024-11-15T16:58:26,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:26,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T16:58:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:26,190 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:58:26,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b945f7e30f14d07898b31c78d5cc844_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689905428/Put/seqid=0 2024-11-15T16:58:26,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741947_1123 (size=12154) 2024-11-15T16:58:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,231 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b945f7e30f14d07898b31c78d5cc844_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b945f7e30f14d07898b31c78d5cc844_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/e24f55e032614a67992a701fb9dd16fb, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:26,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/e24f55e032614a67992a701fb9dd16fb is 175, key is test_row_0/A:col10/1731689905428/Put/seqid=0 2024-11-15T16:58:26,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741948_1124 (size=30955) 2024-11-15T16:58:26,253 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/e24f55e032614a67992a701fb9dd16fb 2024-11-15T16:58:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/a1c74c8e2f6347c785a69694f37472c0 is 50, key is test_row_0/B:col10/1731689905428/Put/seqid=0 2024-11-15T16:58:26,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741949_1125 (size=12001) 2024-11-15T16:58:26,287 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/a1c74c8e2f6347c785a69694f37472c0 2024-11-15T16:58:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/7a081a7b0e984a598c205c7d91467978 is 50, key is test_row_0/C:col10/1731689905428/Put/seqid=0 2024-11-15T16:58:26,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741950_1126 (size=12001) 2024-11-15T16:58:26,318 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/7a081a7b0e984a598c205c7d91467978 2024-11-15T16:58:26,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/e24f55e032614a67992a701fb9dd16fb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/e24f55e032614a67992a701fb9dd16fb 2024-11-15T16:58:26,333 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/e24f55e032614a67992a701fb9dd16fb, entries=150, sequenceid=52, filesize=30.2 K 2024-11-15T16:58:26,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/a1c74c8e2f6347c785a69694f37472c0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/a1c74c8e2f6347c785a69694f37472c0 2024-11-15T16:58:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,344 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/a1c74c8e2f6347c785a69694f37472c0, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T16:58:26,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/7a081a7b0e984a598c205c7d91467978 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/7a081a7b0e984a598c205c7d91467978 2024-11-15T16:58:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,359 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/7a081a7b0e984a598c205c7d91467978, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T16:58:26,360 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 5800e906b79c2ba33b2668d41c684c23 in 171ms, sequenceid=52, compaction requested=true 2024-11-15T16:58:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T16:58:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T16:58:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T16:58:26,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-15T16:58:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.4190 sec 2024-11-15T16:58:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:26,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:26,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:58:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:26,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151578df033afb49bbbba9bd48b7f9fa07_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689906703/Put/seqid=0 2024-11-15T16:58:26,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741952_1128 (size=26798) 2024-11-15T16:58:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689966767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689966771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689966773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689966776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689966776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689966886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689966886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689966886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689966886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689966887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T16:58:27,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689967096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689967097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,098 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:27,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689967097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T16:58:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T16:58:27,102 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:27,103 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:27,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:27,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689967099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689967100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,169 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:27,175 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151578df033afb49bbbba9bd48b7f9fa07_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151578df033afb49bbbba9bd48b7f9fa07_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:27,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/67c788758be846f1be387c5fac7eb446, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:27,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/67c788758be846f1be387c5fac7eb446 is 175, key is test_row_0/A:col10/1731689906703/Put/seqid=0 2024-11-15T16:58:27,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741951_1127 (size=82585) 2024-11-15T16:58:27,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/67c788758be846f1be387c5fac7eb446 2024-11-15T16:58:27,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/108bf8cf27ed4c288a280279089fa4f0 is 50, key is test_row_0/B:col10/1731689906703/Put/seqid=0 2024-11-15T16:58:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T16:58:27,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741953_1129 (size=12001) 2024-11-15T16:58:27,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/108bf8cf27ed4c288a280279089fa4f0 2024-11-15T16:58:27,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/c9492396d14e4a7f8670c6b2ccc69f07 is 50, key is test_row_0/C:col10/1731689906703/Put/seqid=0 2024-11-15T16:58:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741954_1130 (size=12001) 2024-11-15T16:58:27,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/c9492396d14e4a7f8670c6b2ccc69f07 2024-11-15T16:58:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/67c788758be846f1be387c5fac7eb446 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/67c788758be846f1be387c5fac7eb446 2024-11-15T16:58:27,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/67c788758be846f1be387c5fac7eb446, entries=450, sequenceid=64, filesize=80.6 K 2024-11-15T16:58:27,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/108bf8cf27ed4c288a280279089fa4f0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/108bf8cf27ed4c288a280279089fa4f0 2024-11-15T16:58:27,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T16:58:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:27,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/108bf8cf27ed4c288a280279089fa4f0, entries=150, sequenceid=64, filesize=11.7 K 2024-11-15T16:58:27,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/c9492396d14e4a7f8670c6b2ccc69f07 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c9492396d14e4a7f8670c6b2ccc69f07 2024-11-15T16:58:27,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c9492396d14e4a7f8670c6b2ccc69f07, entries=150, sequenceid=64, filesize=11.7 K 2024-11-15T16:58:27,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 5800e906b79c2ba33b2668d41c684c23 in 572ms, sequenceid=64, compaction requested=true 2024-11-15T16:58:27,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:27,282 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:27,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:27,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:27,283 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:27,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:27,284 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 184044 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:27,284 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/A is initiating minor compaction (all files) 2024-11-15T16:58:27,284 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/A in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,285 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f9bb545c5eaf4424998696302510a0c8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d421549dd346463490f1fb85a3c27c3e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/e24f55e032614a67992a701fb9dd16fb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/67c788758be846f1be387c5fac7eb446] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=179.7 K 2024-11-15T16:58:27,285 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,285 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f9bb545c5eaf4424998696302510a0c8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d421549dd346463490f1fb85a3c27c3e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/e24f55e032614a67992a701fb9dd16fb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/67c788758be846f1be387c5fac7eb446] 2024-11-15T16:58:27,286 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:27,286 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9bb545c5eaf4424998696302510a0c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731689904968 2024-11-15T16:58:27,286 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/B is initiating minor compaction (all files) 2024-11-15T16:58:27,286 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/B in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,286 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/12672d8f888b4ce5924e4e5827532482, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1566f598efb64bbcbc96d54b6390bf27, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/a1c74c8e2f6347c785a69694f37472c0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/108bf8cf27ed4c288a280279089fa4f0] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=46.9 K 2024-11-15T16:58:27,287 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d421549dd346463490f1fb85a3c27c3e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731689905041 2024-11-15T16:58:27,287 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e24f55e032614a67992a701fb9dd16fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689905420 2024-11-15T16:58:27,287 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 12672d8f888b4ce5924e4e5827532482, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731689904968 2024-11-15T16:58:27,288 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67c788758be846f1be387c5fac7eb446, keycount=450, bloomtype=ROW, size=80.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731689906632 2024-11-15T16:58:27,288 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1566f598efb64bbcbc96d54b6390bf27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731689905041 2024-11-15T16:58:27,289 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c74c8e2f6347c785a69694f37472c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689905420 2024-11-15T16:58:27,290 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 108bf8cf27ed4c288a280279089fa4f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731689906703 2024-11-15T16:58:27,302 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:27,307 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#B#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:27,308 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/d3fd803964c14ace8938b6ee046432f7 is 50, key is test_row_0/B:col10/1731689906703/Put/seqid=0 2024-11-15T16:58:27,308 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411153a569c32b7d241d99557c82bbb8d9e96_5800e906b79c2ba33b2668d41c684c23 store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:27,315 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411153a569c32b7d241d99557c82bbb8d9e96_5800e906b79c2ba33b2668d41c684c23, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:27,316 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a569c32b7d241d99557c82bbb8d9e96_5800e906b79c2ba33b2668d41c684c23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:27,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741955_1131 (size=12139) 2024-11-15T16:58:27,350 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/d3fd803964c14ace8938b6ee046432f7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/d3fd803964c14ace8938b6ee046432f7 2024-11-15T16:58:27,359 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/B of 5800e906b79c2ba33b2668d41c684c23 into d3fd803964c14ace8938b6ee046432f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:27,359 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:27,359 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/B, priority=12, startTime=1731689907283; duration=0sec 2024-11-15T16:58:27,359 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:27,359 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:B 2024-11-15T16:58:27,359 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:27,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741956_1132 (size=4469) 2024-11-15T16:58:27,362 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:27,363 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/C is initiating minor compaction (all files) 2024-11-15T16:58:27,363 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/C in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,363 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48b1413cf92740788a66eeb4ca621179, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/e2186002e497421a9a40c1f9734c6f7a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/7a081a7b0e984a598c205c7d91467978, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c9492396d14e4a7f8670c6b2ccc69f07] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=46.9 K 2024-11-15T16:58:27,365 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#A#compaction#111 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:27,365 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/205b39f1aa0449598bbd77f67fbe0519 is 175, key is test_row_0/A:col10/1731689906703/Put/seqid=0 2024-11-15T16:58:27,368 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b1413cf92740788a66eeb4ca621179, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731689904968 2024-11-15T16:58:27,370 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e2186002e497421a9a40c1f9734c6f7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731689905041 2024-11-15T16:58:27,370 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a081a7b0e984a598c205c7d91467978, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689905420 2024-11-15T16:58:27,371 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting c9492396d14e4a7f8670c6b2ccc69f07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731689906703 2024-11-15T16:58:27,385 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#C#compaction#113 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:27,385 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/074b3858de4f4c1eb3372db0b2b7402e is 50, key is test_row_0/C:col10/1731689906703/Put/seqid=0 2024-11-15T16:58:27,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741957_1133 (size=31093) 2024-11-15T16:58:27,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741958_1134 (size=12139) 2024-11-15T16:58:27,408 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/205b39f1aa0449598bbd77f67fbe0519 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/205b39f1aa0449598bbd77f67fbe0519 2024-11-15T16:58:27,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T16:58:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,411 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:58:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:27,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:27,416 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/074b3858de4f4c1eb3372db0b2b7402e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/074b3858de4f4c1eb3372db0b2b7402e 2024-11-15T16:58:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T16:58:27,425 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/C of 5800e906b79c2ba33b2668d41c684c23 into 074b3858de4f4c1eb3372db0b2b7402e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:27,425 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:27,425 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/C, priority=12, startTime=1731689907284; duration=0sec 2024-11-15T16:58:27,425 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:27,425 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:C 2024-11-15T16:58:27,425 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/A of 5800e906b79c2ba33b2668d41c684c23 into 205b39f1aa0449598bbd77f67fbe0519(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:27,425 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:27,426 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/A, priority=12, startTime=1731689907282; duration=0sec 2024-11-15T16:58:27,426 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:27,426 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:A 2024-11-15T16:58:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115977d86d7f6be495780391cf7369b3e47_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689906770/Put/seqid=0 2024-11-15T16:58:27,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689967427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689967429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689967432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689967433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689967433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741959_1135 (size=12154) 2024-11-15T16:58:27,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:27,515 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115977d86d7f6be495780391cf7369b3e47_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115977d86d7f6be495780391cf7369b3e47_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:27,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7624c3e3799a4f49a5d55e672f336068, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:27,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7624c3e3799a4f49a5d55e672f336068 is 175, key is test_row_0/A:col10/1731689906770/Put/seqid=0 2024-11-15T16:58:27,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689967537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689967538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689967546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689967546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689967546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741960_1136 (size=30955) 2024-11-15T16:58:27,558 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7624c3e3799a4f49a5d55e672f336068 2024-11-15T16:58:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/9c0d4081fb6f4be9aae38bbb55746a7f is 50, key is test_row_0/B:col10/1731689906770/Put/seqid=0 2024-11-15T16:58:27,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741961_1137 (size=12001) 2024-11-15T16:58:27,588 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/9c0d4081fb6f4be9aae38bbb55746a7f 2024-11-15T16:58:27,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/1e5e50b93d7c452aa3dd1d9979ef190f is 50, key is test_row_0/C:col10/1731689906770/Put/seqid=0 2024-11-15T16:58:27,613 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:58:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741962_1138 (size=12001) 2024-11-15T16:58:27,631 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/1e5e50b93d7c452aa3dd1d9979ef190f 2024-11-15T16:58:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7624c3e3799a4f49a5d55e672f336068 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7624c3e3799a4f49a5d55e672f336068 2024-11-15T16:58:27,647 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7624c3e3799a4f49a5d55e672f336068, entries=150, sequenceid=91, filesize=30.2 K 2024-11-15T16:58:27,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/9c0d4081fb6f4be9aae38bbb55746a7f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/9c0d4081fb6f4be9aae38bbb55746a7f 2024-11-15T16:58:27,657 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/9c0d4081fb6f4be9aae38bbb55746a7f, entries=150, sequenceid=91, filesize=11.7 K 2024-11-15T16:58:27,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/1e5e50b93d7c452aa3dd1d9979ef190f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/1e5e50b93d7c452aa3dd1d9979ef190f 2024-11-15T16:58:27,667 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/1e5e50b93d7c452aa3dd1d9979ef190f, entries=150, sequenceid=91, filesize=11.7 K 2024-11-15T16:58:27,668 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5800e906b79c2ba33b2668d41c684c23 in 257ms, sequenceid=91, compaction requested=false 2024-11-15T16:58:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T16:58:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T16:58:27,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T16:58:27,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 567 msec 2024-11-15T16:58:27,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 581 msec 2024-11-15T16:58:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T16:58:27,726 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:27,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T16:58:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T16:58:27,729 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:27,730 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:27,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:27,749 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:27,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:58:27,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115228adcd2765b4659ae0e9b413dc00cc2_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689907747/Put/seqid=0 2024-11-15T16:58:27,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741963_1139 (size=14594) 2024-11-15T16:58:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689967788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689967790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689967792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689967793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689967795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T16:58:27,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:27,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:27,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:27,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:27,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:27,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689967906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689967906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689967906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689967906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689967906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T16:58:28,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689968116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689968116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689968116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689968117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689968117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:28,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115228adcd2765b4659ae0e9b413dc00cc2_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115228adcd2765b4659ae0e9b413dc00cc2_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:28,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/228af541486b4940a205500fc044c6a7, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:28,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/228af541486b4940a205500fc044c6a7 is 175, key is test_row_0/A:col10/1731689907747/Put/seqid=0 2024-11-15T16:58:28,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741964_1140 (size=39549) 2024-11-15T16:58:28,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T16:58:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689968426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689968426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689968426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689968426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689968426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/228af541486b4940a205500fc044c6a7 2024-11-15T16:58:28,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/875ba3fc20e34b319b2ced1afa00b137 is 50, key is test_row_0/B:col10/1731689907747/Put/seqid=0 2024-11-15T16:58:28,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741965_1141 (size=12001) 2024-11-15T16:58:28,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/875ba3fc20e34b319b2ced1afa00b137 2024-11-15T16:58:28,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/10806aff6dd0426aae6c20cf056d6723 is 50, key is test_row_0/C:col10/1731689907747/Put/seqid=0 2024-11-15T16:58:28,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741966_1142 (size=12001) 2024-11-15T16:58:28,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T16:58:28,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689968937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689968937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689968937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689968937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689968938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:28,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:28,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:29,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/10806aff6dd0426aae6c20cf056d6723 2024-11-15T16:58:29,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/228af541486b4940a205500fc044c6a7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/228af541486b4940a205500fc044c6a7 2024-11-15T16:58:29,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/228af541486b4940a205500fc044c6a7, entries=200, sequenceid=104, filesize=38.6 K 2024-11-15T16:58:29,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/875ba3fc20e34b319b2ced1afa00b137 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/875ba3fc20e34b319b2ced1afa00b137 2024-11-15T16:58:29,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/875ba3fc20e34b319b2ced1afa00b137, entries=150, sequenceid=104, filesize=11.7 K 2024-11-15T16:58:29,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/10806aff6dd0426aae6c20cf056d6723 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/10806aff6dd0426aae6c20cf056d6723 2024-11-15T16:58:29,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/10806aff6dd0426aae6c20cf056d6723, entries=150, sequenceid=104, filesize=11.7 K 2024-11-15T16:58:29,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5800e906b79c2ba33b2668d41c684c23 in 1308ms, sequenceid=104, compaction requested=true 2024-11-15T16:58:29,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:29,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:29,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:29,058 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:29,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:29,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:29,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:29,058 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:29,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:29,060 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:29,060 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/B is initiating minor compaction (all files) 2024-11-15T16:58:29,060 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/B in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:29,060 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/d3fd803964c14ace8938b6ee046432f7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/9c0d4081fb6f4be9aae38bbb55746a7f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/875ba3fc20e34b319b2ced1afa00b137] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=35.3 K 2024-11-15T16:58:29,060 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:29,061 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/A is initiating minor compaction (all files) 2024-11-15T16:58:29,061 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/A in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:29,061 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/205b39f1aa0449598bbd77f67fbe0519, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7624c3e3799a4f49a5d55e672f336068, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/228af541486b4940a205500fc044c6a7] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=99.2 K 2024-11-15T16:58:29,061 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:29,061 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/205b39f1aa0449598bbd77f67fbe0519, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7624c3e3799a4f49a5d55e672f336068, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/228af541486b4940a205500fc044c6a7] 2024-11-15T16:58:29,061 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d3fd803964c14ace8938b6ee046432f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731689906703 2024-11-15T16:58:29,062 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 205b39f1aa0449598bbd77f67fbe0519, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731689906703 2024-11-15T16:58:29,062 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c0d4081fb6f4be9aae38bbb55746a7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731689906770 2024-11-15T16:58:29,064 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7624c3e3799a4f49a5d55e672f336068, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731689906770 2024-11-15T16:58:29,064 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 875ba3fc20e34b319b2ced1afa00b137, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731689907426 2024-11-15T16:58:29,064 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 228af541486b4940a205500fc044c6a7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731689907426 2024-11-15T16:58:29,075 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:29,075 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#B#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:29,076 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/4c041859fa7e40e3b8a58fceace1130c is 50, key is test_row_0/B:col10/1731689907747/Put/seqid=0 2024-11-15T16:58:29,081 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111599c2681822c24368bd8d3ee865936107_5800e906b79c2ba33b2668d41c684c23 store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:29,083 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111599c2681822c24368bd8d3ee865936107_5800e906b79c2ba33b2668d41c684c23, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:29,083 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111599c2681822c24368bd8d3ee865936107_5800e906b79c2ba33b2668d41c684c23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:29,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741967_1143 (size=12241) 2024-11-15T16:58:29,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741968_1144 (size=4469) 2024-11-15T16:58:29,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T16:58:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:29,120 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:58:29,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111560673b99b2ab469688d1f99603399bfc_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689907792/Put/seqid=0 2024-11-15T16:58:29,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741969_1145 (size=12154) 2024-11-15T16:58:29,508 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#A#compaction#121 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:29,509 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/058aba87f9b7420d969ee2ce8e2108ab is 175, key is test_row_0/A:col10/1731689907747/Put/seqid=0 2024-11-15T16:58:29,511 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/4c041859fa7e40e3b8a58fceace1130c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/4c041859fa7e40e3b8a58fceace1130c 2024-11-15T16:58:29,521 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/B of 5800e906b79c2ba33b2668d41c684c23 into 4c041859fa7e40e3b8a58fceace1130c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:29,521 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:29,521 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/B, priority=13, startTime=1731689909058; duration=0sec 2024-11-15T16:58:29,522 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:29,522 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:B 2024-11-15T16:58:29,522 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:29,524 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:29,524 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/C is initiating minor compaction (all files) 2024-11-15T16:58:29,524 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/C in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:29,524 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/074b3858de4f4c1eb3372db0b2b7402e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/1e5e50b93d7c452aa3dd1d9979ef190f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/10806aff6dd0426aae6c20cf056d6723] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=35.3 K 2024-11-15T16:58:29,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741970_1146 (size=31195) 2024-11-15T16:58:29,525 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 074b3858de4f4c1eb3372db0b2b7402e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731689906703 2024-11-15T16:58:29,526 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5e50b93d7c452aa3dd1d9979ef190f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731689906770 2024-11-15T16:58:29,528 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 10806aff6dd0426aae6c20cf056d6723, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731689907426 2024-11-15T16:58:29,533 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/058aba87f9b7420d969ee2ce8e2108ab as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/058aba87f9b7420d969ee2ce8e2108ab 2024-11-15T16:58:29,541 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#C#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:29,542 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/48adf1dc84384fecb0ffa81d837d3ddb is 50, key is test_row_0/C:col10/1731689907747/Put/seqid=0 2024-11-15T16:58:29,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,545 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/A of 5800e906b79c2ba33b2668d41c684c23 into 058aba87f9b7420d969ee2ce8e2108ab(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:29,545 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:29,546 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/A, priority=13, startTime=1731689909057; duration=0sec 2024-11-15T16:58:29,546 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:29,546 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:A 2024-11-15T16:58:29,557 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111560673b99b2ab469688d1f99603399bfc_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560673b99b2ab469688d1f99603399bfc_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:29,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/181644ac886041fa8965b62e848e6d74, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:29,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/181644ac886041fa8965b62e848e6d74 is 175, key is test_row_0/A:col10/1731689907792/Put/seqid=0 2024-11-15T16:58:29,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741971_1147 (size=12241) 2024-11-15T16:58:29,577 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/48adf1dc84384fecb0ffa81d837d3ddb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48adf1dc84384fecb0ffa81d837d3ddb 2024-11-15T16:58:29,585 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/C of 5800e906b79c2ba33b2668d41c684c23 into 48adf1dc84384fecb0ffa81d837d3ddb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:29,585 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:29,585 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/C, priority=13, startTime=1731689909058; duration=0sec 2024-11-15T16:58:29,585 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:29,585 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:C 2024-11-15T16:58:29,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741972_1148 (size=30955) 2024-11-15T16:58:29,590 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/181644ac886041fa8965b62e848e6d74 2024-11-15T16:58:29,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/2709811a3c4845ba8fc0d7972d9cca39 is 50, key is test_row_0/B:col10/1731689907792/Put/seqid=0 2024-11-15T16:58:29,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741973_1149 (size=12001) 2024-11-15T16:58:29,626 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/2709811a3c4845ba8fc0d7972d9cca39 2024-11-15T16:58:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/cf7ca53a740d449e93e7847c09c3c9bb is 50, key is test_row_0/C:col10/1731689907792/Put/seqid=0 2024-11-15T16:58:29,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741974_1150 (size=12001) 2024-11-15T16:58:29,659 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/cf7ca53a740d449e93e7847c09c3c9bb 2024-11-15T16:58:29,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/181644ac886041fa8965b62e848e6d74 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/181644ac886041fa8965b62e848e6d74 2024-11-15T16:58:29,677 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/181644ac886041fa8965b62e848e6d74, entries=150, sequenceid=129, filesize=30.2 K 2024-11-15T16:58:29,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/2709811a3c4845ba8fc0d7972d9cca39 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2709811a3c4845ba8fc0d7972d9cca39 2024-11-15T16:58:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,683 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2709811a3c4845ba8fc0d7972d9cca39, entries=150, sequenceid=129, filesize=11.7 K 2024-11-15T16:58:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/cf7ca53a740d449e93e7847c09c3c9bb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/cf7ca53a740d449e93e7847c09c3c9bb 2024-11-15T16:58:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,690 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/cf7ca53a740d449e93e7847c09c3c9bb, entries=150, sequenceid=129, filesize=11.7 K 2024-11-15T16:58:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,692 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 5800e906b79c2ba33b2668d41c684c23 in 572ms, sequenceid=129, compaction requested=false 2024-11-15T16:58:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T16:58:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T16:58:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T16:58:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9650 sec 2024-11-15T16:58:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9710 sec 2024-11-15T16:58:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T16:58:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,877 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T16:58:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T16:58:29,880 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,881 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:29,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T16:58:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T16:58:30,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:30,034 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-15T16:58:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115632b6a086e76479dbd71ecfef087cdb9_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_1/A:col10/1731689910017/Put/seqid=0 2024-11-15T16:58:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741975_1151 (size=9814) 2024-11-15T16:58:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,083 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115632b6a086e76479dbd71ecfef087cdb9_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115632b6a086e76479dbd71ecfef087cdb9_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/4bfe1761d16b40c29f1b0d355a283e48, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/4bfe1761d16b40c29f1b0d355a283e48 is 175, key is test_row_1/A:col10/1731689910017/Put/seqid=0 2024-11-15T16:58:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741976_1152 (size=22461) 2024-11-15T16:58:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689970142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689970146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689970148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689970150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689970152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T16:58:30,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689970255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689970266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689970266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689970266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689970266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689970465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689970476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689970476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689970476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689970477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,500 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/4bfe1761d16b40c29f1b0d355a283e48 2024-11-15T16:58:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T16:58:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/405b6b1ec19c45359d001f54d850492e is 50, key is test_row_1/B:col10/1731689910017/Put/seqid=0 2024-11-15T16:58:30,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741977_1153 (size=9757) 2024-11-15T16:58:30,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689970776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689970785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689970785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689970796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689970796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:30,919 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/405b6b1ec19c45359d001f54d850492e 2024-11-15T16:58:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/93dbebcf4bab4584b26fcfd523ef3e92 is 50, key is test_row_1/C:col10/1731689910017/Put/seqid=0 2024-11-15T16:58:30,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741978_1154 (size=9757) 2024-11-15T16:58:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T16:58:31,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689971286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689971296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689971296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:31,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689971305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:31,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689971305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:31,347 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/93dbebcf4bab4584b26fcfd523ef3e92 2024-11-15T16:58:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/4bfe1761d16b40c29f1b0d355a283e48 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/4bfe1761d16b40c29f1b0d355a283e48 2024-11-15T16:58:31,362 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/4bfe1761d16b40c29f1b0d355a283e48, entries=100, sequenceid=140, filesize=21.9 K 2024-11-15T16:58:31,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/405b6b1ec19c45359d001f54d850492e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/405b6b1ec19c45359d001f54d850492e 2024-11-15T16:58:31,369 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/405b6b1ec19c45359d001f54d850492e, entries=100, sequenceid=140, filesize=9.5 K 2024-11-15T16:58:31,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/93dbebcf4bab4584b26fcfd523ef3e92 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/93dbebcf4bab4584b26fcfd523ef3e92 2024-11-15T16:58:31,378 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/93dbebcf4bab4584b26fcfd523ef3e92, entries=100, sequenceid=140, filesize=9.5 K 2024-11-15T16:58:31,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 5800e906b79c2ba33b2668d41c684c23 in 1345ms, sequenceid=140, compaction requested=true 2024-11-15T16:58:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T16:58:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T16:58:31,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T16:58:31,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-15T16:58:31,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5100 sec 2024-11-15T16:58:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T16:58:32,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:32,026 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T16:58:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T16:58:32,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:32,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:32,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T16:58:32,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T16:58:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:32,183 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T16:58:32,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159648690bc163443ebaa8d317d85957e7_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689910145/Put/seqid=0 2024-11-15T16:58:32,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741979_1155 (size=12304) 2024-11-15T16:58:32,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:32,203 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159648690bc163443ebaa8d317d85957e7_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159648690bc163443ebaa8d317d85957e7_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1d2f5366570646d8a07b39217d3bb85c, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1d2f5366570646d8a07b39217d3bb85c is 175, key is test_row_0/A:col10/1731689910145/Put/seqid=0 2024-11-15T16:58:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741980_1156 (size=31105) 2024-11-15T16:58:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:32,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:32,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689972316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689972316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689972316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,322 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689972317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,322 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689972326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,327 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T16:58:32,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689972426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689972435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,609 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1d2f5366570646d8a07b39217d3bb85c 2024-11-15T16:58:32,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/eb883a6f33a6452eb07a43bd165ab1ac is 50, key is test_row_0/B:col10/1731689910145/Put/seqid=0 2024-11-15T16:58:32,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741981_1157 (size=12151) 2024-11-15T16:58:32,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689972636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689972646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T16:58:32,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689972946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:32,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689972956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,023 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/eb883a6f33a6452eb07a43bd165ab1ac 2024-11-15T16:58:33,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/b8c92a2b5ee4495cb099335ee0f8f01f is 50, key is test_row_0/C:col10/1731689910145/Put/seqid=0 2024-11-15T16:58:33,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741982_1158 (size=12151) 2024-11-15T16:58:33,037 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/b8c92a2b5ee4495cb099335ee0f8f01f 2024-11-15T16:58:33,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1d2f5366570646d8a07b39217d3bb85c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1d2f5366570646d8a07b39217d3bb85c 2024-11-15T16:58:33,051 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1d2f5366570646d8a07b39217d3bb85c, entries=150, sequenceid=168, filesize=30.4 K 2024-11-15T16:58:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/eb883a6f33a6452eb07a43bd165ab1ac as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/eb883a6f33a6452eb07a43bd165ab1ac 2024-11-15T16:58:33,060 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/eb883a6f33a6452eb07a43bd165ab1ac, entries=150, sequenceid=168, filesize=11.9 K 2024-11-15T16:58:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/b8c92a2b5ee4495cb099335ee0f8f01f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b8c92a2b5ee4495cb099335ee0f8f01f 2024-11-15T16:58:33,068 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b8c92a2b5ee4495cb099335ee0f8f01f, entries=150, sequenceid=168, filesize=11.9 K 2024-11-15T16:58:33,069 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 5800e906b79c2ba33b2668d41c684c23 in 887ms, sequenceid=168, compaction requested=true 2024-11-15T16:58:33,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:33,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:33,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T16:58:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T16:58:33,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T16:58:33,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0410 sec 2024-11-15T16:58:33,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0470 sec 2024-11-15T16:58:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T16:58:33,166 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:33,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T16:58:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T16:58:33,169 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:33,171 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:33,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T16:58:33,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T16:58:33,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:33,324 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T16:58:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111557653ab9311040c3a9a0f3002ee94d9a_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689912313/Put/seqid=0 2024-11-15T16:58:33,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741983_1159 (size=12304) 2024-11-15T16:58:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:33,350 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111557653ab9311040c3a9a0f3002ee94d9a_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111557653ab9311040c3a9a0f3002ee94d9a_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/04799e87be054531bd80309b41346b19, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/04799e87be054531bd80309b41346b19 is 175, key is test_row_0/A:col10/1731689912313/Put/seqid=0 2024-11-15T16:58:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741984_1160 (size=31105) 2024-11-15T16:58:33,362 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/04799e87be054531bd80309b41346b19 2024-11-15T16:58:33,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/db453afb9dff4c35bfda66c1357e0a91 is 50, key is test_row_0/B:col10/1731689912313/Put/seqid=0 2024-11-15T16:58:33,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741985_1161 (size=12151) 2024-11-15T16:58:33,377 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/db453afb9dff4c35bfda66c1357e0a91 2024-11-15T16:58:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/06a36789a4ad458ea0e3c209625e5660 is 50, key is test_row_0/C:col10/1731689912313/Put/seqid=0 2024-11-15T16:58:33,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741986_1162 (size=12151) 2024-11-15T16:58:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:33,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T16:58:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689973527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689973529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689973636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689973636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,793 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/06a36789a4ad458ea0e3c209625e5660 2024-11-15T16:58:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T16:58:33,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/04799e87be054531bd80309b41346b19 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/04799e87be054531bd80309b41346b19 2024-11-15T16:58:33,804 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/04799e87be054531bd80309b41346b19, entries=150, sequenceid=176, filesize=30.4 K 2024-11-15T16:58:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/db453afb9dff4c35bfda66c1357e0a91 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db453afb9dff4c35bfda66c1357e0a91 2024-11-15T16:58:33,810 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db453afb9dff4c35bfda66c1357e0a91, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T16:58:33,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/06a36789a4ad458ea0e3c209625e5660 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/06a36789a4ad458ea0e3c209625e5660 2024-11-15T16:58:33,815 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/06a36789a4ad458ea0e3c209625e5660, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T16:58:33,816 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 5800e906b79c2ba33b2668d41c684c23 in 493ms, sequenceid=176, compaction requested=true 2024-11-15T16:58:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T16:58:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T16:58:33,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T16:58:33,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 647 msec 2024-11-15T16:58:33,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 653 msec 2024-11-15T16:58:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:33,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:33,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-15T16:58:33,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4bb098b3ad24503933f46f0117aa4a2_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689913848/Put/seqid=0 2024-11-15T16:58:33,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689973856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689973863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741987_1163 (size=17284) 2024-11-15T16:58:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689973966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689973966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689974176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689974176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,274 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:34,279 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4bb098b3ad24503933f46f0117aa4a2_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4bb098b3ad24503933f46f0117aa4a2_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:34,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1829b223e1014aff862d09782f78ccc2, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:34,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1829b223e1014aff862d09782f78ccc2 is 175, key is test_row_0/A:col10/1731689913848/Put/seqid=0 2024-11-15T16:58:34,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741988_1164 (size=48389) 2024-11-15T16:58:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T16:58:34,305 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:34,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-15T16:58:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T16:58:34,308 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:34,308 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:34,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:34,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689974335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689974335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,338 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,338 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689974346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,349 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T16:58:34,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689974486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689974486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:34,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T16:58:34,686 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1829b223e1014aff862d09782f78ccc2 2024-11-15T16:58:34,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/e29e96daab0c45a6aab0d8f896cffc2f is 50, key is test_row_0/B:col10/1731689913848/Put/seqid=0 2024-11-15T16:58:34,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741989_1165 (size=12151) 2024-11-15T16:58:34,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:34,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:34,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:34,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:34,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:34,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T16:58:34,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689974995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689974995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:35,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:35,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:35,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/e29e96daab0c45a6aab0d8f896cffc2f 2024-11-15T16:58:35,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/8318bbc194a24561bf705055609b174e is 50, key is test_row_0/C:col10/1731689913848/Put/seqid=0 2024-11-15T16:58:35,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741990_1166 (size=12151) 2024-11-15T16:58:35,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:35,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:35,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:35,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T16:58:35,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/8318bbc194a24561bf705055609b174e 2024-11-15T16:58:35,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/1829b223e1014aff862d09782f78ccc2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1829b223e1014aff862d09782f78ccc2 2024-11-15T16:58:35,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1829b223e1014aff862d09782f78ccc2, entries=250, sequenceid=206, filesize=47.3 K 2024-11-15T16:58:35,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/e29e96daab0c45a6aab0d8f896cffc2f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/e29e96daab0c45a6aab0d8f896cffc2f 2024-11-15T16:58:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/e29e96daab0c45a6aab0d8f896cffc2f, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T16:58:35,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/8318bbc194a24561bf705055609b174e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8318bbc194a24561bf705055609b174e 2024-11-15T16:58:35,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:35,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:35,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8318bbc194a24561bf705055609b174e, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T16:58:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:35,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 5800e906b79c2ba33b2668d41c684c23 in 1691ms, sequenceid=206, compaction requested=true 2024-11-15T16:58:35,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,540 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T16:58:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:35,541 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T16:58:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,545 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70452 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T16:58:35,545 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/B is initiating minor compaction (all files) 2024-11-15T16:58:35,545 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/B in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,545 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/4c041859fa7e40e3b8a58fceace1130c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2709811a3c4845ba8fc0d7972d9cca39, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/405b6b1ec19c45359d001f54d850492e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/eb883a6f33a6452eb07a43bd165ab1ac, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db453afb9dff4c35bfda66c1357e0a91, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/e29e96daab0c45a6aab0d8f896cffc2f] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=68.8 K 2024-11-15T16:58:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,545 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195210 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T16:58:35,545 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/A is initiating minor compaction (all files) 2024-11-15T16:58:35,546 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/A in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,546 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/058aba87f9b7420d969ee2ce8e2108ab, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/181644ac886041fa8965b62e848e6d74, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/4bfe1761d16b40c29f1b0d355a283e48, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1d2f5366570646d8a07b39217d3bb85c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/04799e87be054531bd80309b41346b19, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1829b223e1014aff862d09782f78ccc2] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=190.6 K 2024-11-15T16:58:35,546 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,546 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/058aba87f9b7420d969ee2ce8e2108ab, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/181644ac886041fa8965b62e848e6d74, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/4bfe1761d16b40c29f1b0d355a283e48, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1d2f5366570646d8a07b39217d3bb85c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/04799e87be054531bd80309b41346b19, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1829b223e1014aff862d09782f78ccc2] 2024-11-15T16:58:35,546 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c041859fa7e40e3b8a58fceace1130c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731689907426 2024-11-15T16:58:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,547 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 058aba87f9b7420d969ee2ce8e2108ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731689907426 2024-11-15T16:58:35,547 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2709811a3c4845ba8fc0d7972d9cca39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731689907781 2024-11-15T16:58:35,548 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 181644ac886041fa8965b62e848e6d74, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731689907781 2024-11-15T16:58:35,548 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405b6b1ec19c45359d001f54d850492e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731689910011 2024-11-15T16:58:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,548 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfe1761d16b40c29f1b0d355a283e48, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731689910011 2024-11-15T16:58:35,548 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb883a6f33a6452eb07a43bd165ab1ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731689910145 2024-11-15T16:58:35,549 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2f5366570646d8a07b39217d3bb85c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731689910145 2024-11-15T16:58:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,549 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting db453afb9dff4c35bfda66c1357e0a91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731689912310 2024-11-15T16:58:35,550 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 04799e87be054531bd80309b41346b19, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731689912310 2024-11-15T16:58:35,550 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e29e96daab0c45a6aab0d8f896cffc2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689913520 2024-11-15T16:58:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,550 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1829b223e1014aff862d09782f78ccc2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689913520 2024-11-15T16:58:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,574 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,584 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115053343ef552b4bcb9f72a3cd7eaa5334_5800e906b79c2ba33b2668d41c684c23 store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,591 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#B#compaction#139 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,592 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/2cfcc5746ee24f0f8360ab8777c33dce is 50, key is test_row_0/B:col10/1731689913848/Put/seqid=0 2024-11-15T16:58:35,593 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115053343ef552b4bcb9f72a3cd7eaa5334_5800e906b79c2ba33b2668d41c684c23, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:35,593 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115053343ef552b4bcb9f72a3cd7eaa5334_5800e906b79c2ba33b2668d41c684c23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741991_1167 (size=12595) 2024-11-15T16:58:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,618 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/2cfcc5746ee24f0f8360ab8777c33dce as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2cfcc5746ee24f0f8360ab8777c33dce 2024-11-15T16:58:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,627 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/B of 5800e906b79c2ba33b2668d41c684c23 into 2cfcc5746ee24f0f8360ab8777c33dce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:35,627 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:35,627 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/B, priority=10, startTime=1731689915541; duration=0sec 2024-11-15T16:58:35,629 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:35,629 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:B 2024-11-15T16:58:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,629 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T16:58:35,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741992_1168 (size=4469) 2024-11-15T16:58:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,633 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70452 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T16:58:35,633 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/C is initiating minor compaction (all files) 2024-11-15T16:58:35,633 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/C in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,633 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48adf1dc84384fecb0ffa81d837d3ddb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/cf7ca53a740d449e93e7847c09c3c9bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/93dbebcf4bab4584b26fcfd523ef3e92, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b8c92a2b5ee4495cb099335ee0f8f01f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/06a36789a4ad458ea0e3c209625e5660, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8318bbc194a24561bf705055609b174e] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=68.8 K 2024-11-15T16:58:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,633 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48adf1dc84384fecb0ffa81d837d3ddb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731689907426 2024-11-15T16:58:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,634 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7ca53a740d449e93e7847c09c3c9bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731689907781 2024-11-15T16:58:35,634 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93dbebcf4bab4584b26fcfd523ef3e92, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731689910011 2024-11-15T16:58:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8c92a2b5ee4495cb099335ee0f8f01f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731689910145 2024-11-15T16:58:35,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a36789a4ad458ea0e3c209625e5660, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731689912310 2024-11-15T16:58:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,636 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8318bbc194a24561bf705055609b174e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689913520 2024-11-15T16:58:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,660 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#C#compaction#140 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,661 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/d7c2ae6375db4a3580210acd36d7c2a9 is 50, key is test_row_0/C:col10/1731689913848/Put/seqid=0 2024-11-15T16:58:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741993_1169 (size=12595) 2024-11-15T16:58:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,691 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/d7c2ae6375db4a3580210acd36d7c2a9 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/d7c2ae6375db4a3580210acd36d7c2a9 2024-11-15T16:58:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T16:58:35,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,692 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-15T16:58:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151e3822d5193145438f6b04b23e8f2cdb_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689913861/Put/seqid=0 2024-11-15T16:58:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,705 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/C of 5800e906b79c2ba33b2668d41c684c23 into d7c2ae6375db4a3580210acd36d7c2a9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:35,706 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:35,706 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/C, priority=10, startTime=1731689915541; duration=0sec 2024-11-15T16:58:35,706 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:35,706 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:C 2024-11-15T16:58:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741994_1170 (size=9814) 2024-11-15T16:58:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,736 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151e3822d5193145438f6b04b23e8f2cdb_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151e3822d5193145438f6b04b23e8f2cdb_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/50a7cdc103be42d28c4fa08a1daa8188, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/50a7cdc103be42d28c4fa08a1daa8188 is 175, key is test_row_0/A:col10/1731689913861/Put/seqid=0 2024-11-15T16:58:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741995_1171 (size=22461) 2024-11-15T16:58:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,759 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/50a7cdc103be42d28c4fa08a1daa8188 2024-11-15T16:58:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/07c6376a66d64dfb9a24148120e3e210 is 50, key is test_row_0/B:col10/1731689913861/Put/seqid=0 2024-11-15T16:58:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741996_1172 (size=9757) 2024-11-15T16:58:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,787 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/07c6376a66d64dfb9a24148120e3e210 2024-11-15T16:58:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/2abfdcaa690f4269b94ffeede5dad400 is 50, key is test_row_0/C:col10/1731689913861/Put/seqid=0 2024-11-15T16:58:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741997_1173 (size=9757) 2024-11-15T16:58:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,827 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/2abfdcaa690f4269b94ffeede5dad400 2024-11-15T16:58:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/50a7cdc103be42d28c4fa08a1daa8188 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/50a7cdc103be42d28c4fa08a1daa8188 2024-11-15T16:58:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,858 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/50a7cdc103be42d28c4fa08a1daa8188, entries=100, sequenceid=213, filesize=21.9 K 2024-11-15T16:58:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/07c6376a66d64dfb9a24148120e3e210 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/07c6376a66d64dfb9a24148120e3e210 2024-11-15T16:58:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,870 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/07c6376a66d64dfb9a24148120e3e210, entries=100, sequenceid=213, filesize=9.5 K 2024-11-15T16:58:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/2abfdcaa690f4269b94ffeede5dad400 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2abfdcaa690f4269b94ffeede5dad400 2024-11-15T16:58:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2abfdcaa690f4269b94ffeede5dad400, entries=100, sequenceid=213, filesize=9.5 K 2024-11-15T16:58:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,879 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 5800e906b79c2ba33b2668d41c684c23 in 187ms, sequenceid=213, compaction requested=false 2024-11-15T16:58:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-15T16:58:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-15T16:58:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T16:58:35,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-11-15T16:58:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5790 sec 2024-11-15T16:58:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,031 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,032 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/ae5a1e69df46424a8284244e4dae3528 is 175, key is test_row_0/A:col10/1731689913848/Put/seqid=0 2024-11-15T16:58:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741998_1174 (size=31549) 2024-11-15T16:58:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,057 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/ae5a1e69df46424a8284244e4dae3528 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/ae5a1e69df46424a8284244e4dae3528 2024-11-15T16:58:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,065 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/A of 5800e906b79c2ba33b2668d41c684c23 into ae5a1e69df46424a8284244e4dae3528(size=30.8 K), total size for store is 52.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:36,065 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:36,065 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/A, priority=10, startTime=1731689915540; duration=0sec 2024-11-15T16:58:36,065 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:36,065 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:A 2024-11-15T16:58:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,070 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:58:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111579627a36f0a845ef88274945e8a933c0_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689916066/Put/seqid=0 2024-11-15T16:58:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742000_1176 (size=24758) 2024-11-15T16:58:36,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689976129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689976130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689976235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689976235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,366 INFO [master/a9bde5b3a1b5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T16:58:36,366 INFO [master/a9bde5b3a1b5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T16:58:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689976446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689976446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T16:58:36,456 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:36,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-15T16:58:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T16:58:36,458 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:36,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:36,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:36,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:36,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111579627a36f0a845ef88274945e8a933c0_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579627a36f0a845ef88274945e8a933c0_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:36,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d5e9cfff1d93402695aa1e78ea14ea77, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:36,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d5e9cfff1d93402695aa1e78ea14ea77 is 175, key is test_row_0/A:col10/1731689916066/Put/seqid=0 2024-11-15T16:58:36,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741999_1175 (size=74395) 2024-11-15T16:58:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T16:58:36,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T16:58:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:36,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689976756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689976756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:36,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T16:58:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:36,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T16:58:36,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T16:58:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:36,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:36,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:36,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:36,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d5e9cfff1d93402695aa1e78ea14ea77 2024-11-15T16:58:36,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/64ad38fe15d8426aa7fe1b7e51911650 is 50, key is test_row_0/B:col10/1731689916066/Put/seqid=0 2024-11-15T16:58:36,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742001_1177 (size=12151) 2024-11-15T16:58:36,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/64ad38fe15d8426aa7fe1b7e51911650 2024-11-15T16:58:36,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/88f46e3473a5417887fedec5c090c589 is 50, key is test_row_0/C:col10/1731689916066/Put/seqid=0 2024-11-15T16:58:36,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742002_1178 (size=12151) 2024-11-15T16:58:37,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T16:58:37,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:37,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T16:58:37,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T16:58:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689977265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:37,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689977265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:37,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/88f46e3473a5417887fedec5c090c589 2024-11-15T16:58:37,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/d5e9cfff1d93402695aa1e78ea14ea77 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d5e9cfff1d93402695aa1e78ea14ea77 2024-11-15T16:58:37,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d5e9cfff1d93402695aa1e78ea14ea77, entries=400, sequenceid=226, filesize=72.7 K 2024-11-15T16:58:37,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/64ad38fe15d8426aa7fe1b7e51911650 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/64ad38fe15d8426aa7fe1b7e51911650 2024-11-15T16:58:37,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T16:58:37,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:37,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/64ad38fe15d8426aa7fe1b7e51911650, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T16:58:37,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/88f46e3473a5417887fedec5c090c589 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/88f46e3473a5417887fedec5c090c589 2024-11-15T16:58:37,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/88f46e3473a5417887fedec5c090c589, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T16:58:37,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5800e906b79c2ba33b2668d41c684c23 in 1320ms, sequenceid=226, compaction requested=true 2024-11-15T16:58:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:37,390 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:37,390 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:37,392 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:37,392 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:37,392 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/B is initiating minor compaction (all files) 2024-11-15T16:58:37,392 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/A is initiating minor compaction (all files) 2024-11-15T16:58:37,392 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/B in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,392 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/A in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,392 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2cfcc5746ee24f0f8360ab8777c33dce, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/07c6376a66d64dfb9a24148120e3e210, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/64ad38fe15d8426aa7fe1b7e51911650] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=33.7 K 2024-11-15T16:58:37,392 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/ae5a1e69df46424a8284244e4dae3528, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/50a7cdc103be42d28c4fa08a1daa8188, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d5e9cfff1d93402695aa1e78ea14ea77] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=125.4 K 2024-11-15T16:58:37,392 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,392 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/ae5a1e69df46424a8284244e4dae3528, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/50a7cdc103be42d28c4fa08a1daa8188, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d5e9cfff1d93402695aa1e78ea14ea77] 2024-11-15T16:58:37,392 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cfcc5746ee24f0f8360ab8777c33dce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689913520 2024-11-15T16:58:37,393 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5a1e69df46424a8284244e4dae3528, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689913520 2024-11-15T16:58:37,393 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c6376a66d64dfb9a24148120e3e210, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731689913853 2024-11-15T16:58:37,393 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a7cdc103be42d28c4fa08a1daa8188, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731689913853 2024-11-15T16:58:37,393 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ad38fe15d8426aa7fe1b7e51911650, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731689916049 2024-11-15T16:58:37,394 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5e9cfff1d93402695aa1e78ea14ea77, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731689916025 2024-11-15T16:58:37,402 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:37,402 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/1c2074e07ad84332a91e2f2cf9d06dcc is 50, key is test_row_0/B:col10/1731689916066/Put/seqid=0 2024-11-15T16:58:37,407 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:37,413 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115884e699168b64dd5adbcd55360885e17_5800e906b79c2ba33b2668d41c684c23 store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:37,415 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115884e699168b64dd5adbcd55360885e17_5800e906b79c2ba33b2668d41c684c23, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:37,415 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115884e699168b64dd5adbcd55360885e17_5800e906b79c2ba33b2668d41c684c23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:37,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742004_1180 (size=4469) 2024-11-15T16:58:37,427 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#A#compaction#148 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:37,428 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/c4bb161053084084b577bbfea3127b70 is 175, key is test_row_0/A:col10/1731689916066/Put/seqid=0 2024-11-15T16:58:37,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742003_1179 (size=12697) 2024-11-15T16:58:37,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742005_1181 (size=31651) 2024-11-15T16:58:37,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T16:58:37,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,534 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:58:37,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f518a05ad5fb4190a833472c676de433_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689916120/Put/seqid=0 2024-11-15T16:58:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742006_1182 (size=12304) 2024-11-15T16:58:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T16:58:37,837 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/1c2074e07ad84332a91e2f2cf9d06dcc as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1c2074e07ad84332a91e2f2cf9d06dcc 2024-11-15T16:58:37,843 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/B of 5800e906b79c2ba33b2668d41c684c23 into 1c2074e07ad84332a91e2f2cf9d06dcc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:37,843 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:37,843 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/B, priority=13, startTime=1731689917390; duration=0sec 2024-11-15T16:58:37,844 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:37,844 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:B 2024-11-15T16:58:37,844 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:37,844 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/c4bb161053084084b577bbfea3127b70 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c4bb161053084084b577bbfea3127b70 2024-11-15T16:58:37,846 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:37,846 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/C is initiating minor compaction (all files) 2024-11-15T16:58:37,846 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/C in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:37,846 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/d7c2ae6375db4a3580210acd36d7c2a9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2abfdcaa690f4269b94ffeede5dad400, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/88f46e3473a5417887fedec5c090c589] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=33.7 K 2024-11-15T16:58:37,846 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c2ae6375db4a3580210acd36d7c2a9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689913520 2024-11-15T16:58:37,847 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2abfdcaa690f4269b94ffeede5dad400, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731689913853 2024-11-15T16:58:37,847 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f46e3473a5417887fedec5c090c589, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731689916049 2024-11-15T16:58:37,850 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/A of 5800e906b79c2ba33b2668d41c684c23 into c4bb161053084084b577bbfea3127b70(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:37,850 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:37,850 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/A, priority=13, startTime=1731689917390; duration=0sec 2024-11-15T16:58:37,850 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:37,850 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:A 2024-11-15T16:58:37,859 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:37,859 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/c254a164bdc24536854c107b6d0ac748 is 50, key is test_row_0/C:col10/1731689916066/Put/seqid=0 2024-11-15T16:58:37,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742007_1183 (size=12697) 2024-11-15T16:58:37,873 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/c254a164bdc24536854c107b6d0ac748 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c254a164bdc24536854c107b6d0ac748 2024-11-15T16:58:37,880 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/C of 5800e906b79c2ba33b2668d41c684c23 into c254a164bdc24536854c107b6d0ac748(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:37,880 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:37,880 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/C, priority=13, startTime=1731689917390; duration=0sec 2024-11-15T16:58:37,881 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:37,881 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:C 2024-11-15T16:58:37,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:37,959 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f518a05ad5fb4190a833472c676de433_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f518a05ad5fb4190a833472c676de433_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:37,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/8c3d4d7895324bf1b6c52cb06681b143, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:37,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/8c3d4d7895324bf1b6c52cb06681b143 is 175, key is test_row_0/A:col10/1731689916120/Put/seqid=0 2024-11-15T16:58:37,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742008_1184 (size=31105) 2024-11-15T16:58:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:38,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689978289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689978291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37300 deadline: 1731689978366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,367 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/8c3d4d7895324bf1b6c52cb06681b143 2024-11-15T16:58:38,367 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/959cde74cccd451aa614563683198f20 is 50, key is test_row_0/B:col10/1731689916120/Put/seqid=0 2024-11-15T16:58:38,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37248 deadline: 1731689978376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,377 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742009_1185 (size=12151) 2024-11-15T16:58:38,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37260 deadline: 1731689978395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689978395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,398 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5800e906b79c2ba33b2668d41c684c23 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:58:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689978405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T16:58:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689978606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689978615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/959cde74cccd451aa614563683198f20 2024-11-15T16:58:38,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/3938dc3232d44e4abd90872dcb081dfe is 50, key is test_row_0/C:col10/1731689916120/Put/seqid=0 2024-11-15T16:58:38,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742010_1186 (size=12151) 2024-11-15T16:58:38,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689978915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689978925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:39,194 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/3938dc3232d44e4abd90872dcb081dfe 2024-11-15T16:58:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/8c3d4d7895324bf1b6c52cb06681b143 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/8c3d4d7895324bf1b6c52cb06681b143 2024-11-15T16:58:39,204 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/8c3d4d7895324bf1b6c52cb06681b143, entries=150, sequenceid=251, filesize=30.4 K 2024-11-15T16:58:39,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/959cde74cccd451aa614563683198f20 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/959cde74cccd451aa614563683198f20 2024-11-15T16:58:39,209 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/959cde74cccd451aa614563683198f20, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T16:58:39,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/3938dc3232d44e4abd90872dcb081dfe as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/3938dc3232d44e4abd90872dcb081dfe 2024-11-15T16:58:39,215 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/3938dc3232d44e4abd90872dcb081dfe, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T16:58:39,216 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5800e906b79c2ba33b2668d41c684c23 in 1682ms, sequenceid=251, compaction requested=false 2024-11-15T16:58:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-15T16:58:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-15T16:58:39,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T16:58:39,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7590 sec 2024-11-15T16:58:39,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7640 sec 2024-11-15T16:58:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:39,427 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:39,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T16:58:39,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d178dfd5da6742cd941e0cbafefd0162_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689919426/Put/seqid=0 2024-11-15T16:58:39,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742011_1187 (size=14994) 2024-11-15T16:58:39,447 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:39,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d178dfd5da6742cd941e0cbafefd0162_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d178dfd5da6742cd941e0cbafefd0162_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:39,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/c40726b710574f8fad74047bb9a481c5, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:39,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/c40726b710574f8fad74047bb9a481c5 is 175, key is test_row_0/A:col10/1731689919426/Put/seqid=0 2024-11-15T16:58:39,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742012_1188 (size=39949) 2024-11-15T16:58:39,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/c40726b710574f8fad74047bb9a481c5 2024-11-15T16:58:39,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/873884c633f0468f805bff454d8bd892 is 50, key is test_row_0/B:col10/1731689919426/Put/seqid=0 2024-11-15T16:58:39,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742013_1189 (size=12301) 2024-11-15T16:58:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689979511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689979513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:39,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689979625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:39,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689979625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:39,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:39,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689979836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:39,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689979836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:39,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/873884c633f0468f805bff454d8bd892 2024-11-15T16:58:39,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/aa1251611c5b4a18af2e42d69979b5d5 is 50, key is test_row_0/C:col10/1731689919426/Put/seqid=0 2024-11-15T16:58:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742014_1190 (size=12301) 2024-11-15T16:58:39,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/aa1251611c5b4a18af2e42d69979b5d5 2024-11-15T16:58:39,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/c40726b710574f8fad74047bb9a481c5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c40726b710574f8fad74047bb9a481c5 2024-11-15T16:58:39,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c40726b710574f8fad74047bb9a481c5, entries=200, sequenceid=266, filesize=39.0 K 2024-11-15T16:58:39,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/873884c633f0468f805bff454d8bd892 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/873884c633f0468f805bff454d8bd892 2024-11-15T16:58:39,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/873884c633f0468f805bff454d8bd892, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T16:58:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/aa1251611c5b4a18af2e42d69979b5d5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/aa1251611c5b4a18af2e42d69979b5d5 2024-11-15T16:58:39,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/aa1251611c5b4a18af2e42d69979b5d5, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T16:58:39,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5800e906b79c2ba33b2668d41c684c23 in 495ms, sequenceid=266, compaction requested=true 2024-11-15T16:58:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:39,923 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:39,923 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:39,924 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:39,924 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/A is initiating minor compaction (all files) 2024-11-15T16:58:39,924 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/A in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:39,924 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c4bb161053084084b577bbfea3127b70, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/8c3d4d7895324bf1b6c52cb06681b143, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c40726b710574f8fad74047bb9a481c5] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=100.3 K 2024-11-15T16:58:39,924 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:39,924 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:39,924 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/B is initiating minor compaction (all files) 2024-11-15T16:58:39,924 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c4bb161053084084b577bbfea3127b70, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/8c3d4d7895324bf1b6c52cb06681b143, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c40726b710574f8fad74047bb9a481c5] 2024-11-15T16:58:39,924 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/B in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:39,924 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1c2074e07ad84332a91e2f2cf9d06dcc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/959cde74cccd451aa614563683198f20, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/873884c633f0468f805bff454d8bd892] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=36.3 K 2024-11-15T16:58:39,925 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4bb161053084084b577bbfea3127b70, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731689916049 2024-11-15T16:58:39,925 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c2074e07ad84332a91e2f2cf9d06dcc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731689916049 2024-11-15T16:58:39,926 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c3d4d7895324bf1b6c52cb06681b143, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731689916120 2024-11-15T16:58:39,926 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 959cde74cccd451aa614563683198f20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731689916120 2024-11-15T16:58:39,926 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c40726b710574f8fad74047bb9a481c5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731689918280 2024-11-15T16:58:39,926 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 873884c633f0468f805bff454d8bd892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731689918280 2024-11-15T16:58:39,935 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:39,937 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:39,938 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/06613918a9ad4895ba8167d95baa9dd7 is 50, key is test_row_0/B:col10/1731689919426/Put/seqid=0 2024-11-15T16:58:39,938 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ccdf2637de014409b327fbb2b6c483f1_5800e906b79c2ba33b2668d41c684c23 store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:39,940 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ccdf2637de014409b327fbb2b6c483f1_5800e906b79c2ba33b2668d41c684c23, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:39,941 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ccdf2637de014409b327fbb2b6c483f1_5800e906b79c2ba33b2668d41c684c23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:39,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742015_1191 (size=12949) 2024-11-15T16:58:39,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742016_1192 (size=4469) 2024-11-15T16:58:39,960 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#A#compaction#156 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:39,961 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7bef2fec9de5477a8fba0a1db21bda7f is 175, key is test_row_0/A:col10/1731689919426/Put/seqid=0 2024-11-15T16:58:39,963 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/06613918a9ad4895ba8167d95baa9dd7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/06613918a9ad4895ba8167d95baa9dd7 2024-11-15T16:58:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742017_1193 (size=31903) 2024-11-15T16:58:39,974 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/B of 5800e906b79c2ba33b2668d41c684c23 into 06613918a9ad4895ba8167d95baa9dd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:39,974 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7bef2fec9de5477a8fba0a1db21bda7f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7bef2fec9de5477a8fba0a1db21bda7f 2024-11-15T16:58:39,974 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:39,974 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/B, priority=13, startTime=1731689919923; duration=0sec 2024-11-15T16:58:39,974 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:39,975 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:B 2024-11-15T16:58:39,975 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:39,978 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:39,978 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/C is initiating minor compaction (all files) 2024-11-15T16:58:39,978 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/C in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:39,978 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c254a164bdc24536854c107b6d0ac748, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/3938dc3232d44e4abd90872dcb081dfe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/aa1251611c5b4a18af2e42d69979b5d5] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=36.3 K 2024-11-15T16:58:39,979 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting c254a164bdc24536854c107b6d0ac748, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731689916049 2024-11-15T16:58:39,981 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 3938dc3232d44e4abd90872dcb081dfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731689916120 2024-11-15T16:58:39,981 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting aa1251611c5b4a18af2e42d69979b5d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731689918280 2024-11-15T16:58:39,982 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/A of 5800e906b79c2ba33b2668d41c684c23 into 7bef2fec9de5477a8fba0a1db21bda7f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:39,982 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:39,982 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/A, priority=13, startTime=1731689919923; duration=0sec 2024-11-15T16:58:39,982 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:39,982 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:A 2024-11-15T16:58:39,990 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:39,990 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/b39f5d484afb4b63b958e1253b814167 is 50, key is test_row_0/C:col10/1731689919426/Put/seqid=0 2024-11-15T16:58:40,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742018_1194 (size=12949) 2024-11-15T16:58:40,012 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/b39f5d484afb4b63b958e1253b814167 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b39f5d484afb4b63b958e1253b814167 2024-11-15T16:58:40,022 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/C of 5800e906b79c2ba33b2668d41c684c23 into b39f5d484afb4b63b958e1253b814167(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:40,022 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:40,023 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/C, priority=13, startTime=1731689919923; duration=0sec 2024-11-15T16:58:40,023 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:40,023 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:C 2024-11-15T16:58:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:40,149 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:40,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T16:58:40,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d609f2a3f86a41dd850664a5101af276_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689919511/Put/seqid=0 2024-11-15T16:58:40,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742019_1195 (size=12454) 2024-11-15T16:58:40,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689980168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689980170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689980276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689980286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689980486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689980496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,566 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:40,571 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d609f2a3f86a41dd850664a5101af276_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d609f2a3f86a41dd850664a5101af276_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:40,572 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:40,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f is 175, key is test_row_0/A:col10/1731689919511/Put/seqid=0 2024-11-15T16:58:40,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742020_1196 (size=31255) 2024-11-15T16:58:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T16:58:40,616 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:40,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-15T16:58:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T16:58:40,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:40,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:40,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T16:58:40,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:40,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:40,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689980795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689980805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:40,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:40,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T16:58:40,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f 2024-11-15T16:58:40,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/7b5af78ddbf941108348a6807a17d9b4 is 50, key is test_row_0/B:col10/1731689919511/Put/seqid=0 2024-11-15T16:58:41,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742021_1197 (size=12301) 2024-11-15T16:58:41,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T16:58:41,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689981305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:41,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689981316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:41,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/7b5af78ddbf941108348a6807a17d9b4 2024-11-15T16:58:41,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/00030984a2d940d0b819cd9016189580 is 50, key is test_row_0/C:col10/1731689919511/Put/seqid=0 2024-11-15T16:58:41,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742022_1198 (size=12301) 2024-11-15T16:58:41,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T16:58:41,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/00030984a2d940d0b819cd9016189580 2024-11-15T16:58:41,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f 2024-11-15T16:58:41,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f, entries=150, sequenceid=295, filesize=30.5 K 2024-11-15T16:58:41,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/7b5af78ddbf941108348a6807a17d9b4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/7b5af78ddbf941108348a6807a17d9b4 2024-11-15T16:58:41,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/7b5af78ddbf941108348a6807a17d9b4, entries=150, sequenceid=295, filesize=12.0 K 2024-11-15T16:58:41,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/00030984a2d940d0b819cd9016189580 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/00030984a2d940d0b819cd9016189580 2024-11-15T16:58:41,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:41,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:41,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:41,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/00030984a2d940d0b819cd9016189580, entries=150, sequenceid=295, filesize=12.0 K 2024-11-15T16:58:41,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5800e906b79c2ba33b2668d41c684c23 in 1706ms, sequenceid=295, compaction requested=false 2024-11-15T16:58:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:42,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T16:58:42,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:42,002 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T16:58:42,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151d78c20fc4944f05998d1d94d5469087_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689920158/Put/seqid=0 2024-11-15T16:58:42,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742023_1199 (size=12454) 2024-11-15T16:58:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:42,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. as already flushing 2024-11-15T16:58:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689982368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689982371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:42,419 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151d78c20fc4944f05998d1d94d5469087_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151d78c20fc4944f05998d1d94d5469087_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:42,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7ef3bf050bf64291924840558956fcd6, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:42,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7ef3bf050bf64291924840558956fcd6 is 175, key is test_row_0/A:col10/1731689920158/Put/seqid=0 2024-11-15T16:58:42,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742024_1200 (size=31255) 2024-11-15T16:58:42,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689982475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689982475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:42,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689982685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689982685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T16:58:42,826 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7ef3bf050bf64291924840558956fcd6 2024-11-15T16:58:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/bf2e15a92a6f4288a1a0d855e0c79714 is 50, key is test_row_0/B:col10/1731689920158/Put/seqid=0 2024-11-15T16:58:42,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742025_1201 (size=12301) 2024-11-15T16:58:42,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689982996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689982996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:43,240 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/bf2e15a92a6f4288a1a0d855e0c79714 2024-11-15T16:58:43,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/fed9fec55c544fa2b2bfd51b96d736d6 is 50, key is test_row_0/C:col10/1731689920158/Put/seqid=0 2024-11-15T16:58:43,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742026_1202 (size=12301) 2024-11-15T16:58:43,252 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/fed9fec55c544fa2b2bfd51b96d736d6 2024-11-15T16:58:43,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/7ef3bf050bf64291924840558956fcd6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7ef3bf050bf64291924840558956fcd6 2024-11-15T16:58:43,262 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7ef3bf050bf64291924840558956fcd6, entries=150, sequenceid=305, filesize=30.5 K 2024-11-15T16:58:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/bf2e15a92a6f4288a1a0d855e0c79714 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/bf2e15a92a6f4288a1a0d855e0c79714 2024-11-15T16:58:43,267 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/bf2e15a92a6f4288a1a0d855e0c79714, entries=150, sequenceid=305, filesize=12.0 K 2024-11-15T16:58:43,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/fed9fec55c544fa2b2bfd51b96d736d6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/fed9fec55c544fa2b2bfd51b96d736d6 2024-11-15T16:58:43,273 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/fed9fec55c544fa2b2bfd51b96d736d6, entries=150, sequenceid=305, filesize=12.0 K 2024-11-15T16:58:43,274 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 5800e906b79c2ba33b2668d41c684c23 in 1272ms, sequenceid=305, compaction requested=true 2024-11-15T16:58:43,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:43,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:43,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-15T16:58:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-15T16:58:43,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T16:58:43,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6550 sec 2024-11-15T16:58:43,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.6620 sec 2024-11-15T16:58:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:43,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T16:58:43,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150928cace83114559a55fe6c6f135f5fa_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689922360/Put/seqid=0 2024-11-15T16:58:43,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689983517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:43,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689983518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:43,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742027_1203 (size=17534) 2024-11-15T16:58:43,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689983626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:43,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689983626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689983835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689983836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:43,926 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:43,931 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150928cace83114559a55fe6c6f135f5fa_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150928cace83114559a55fe6c6f135f5fa_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:43,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/b911028f86854952bd23a2a3717ca043, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:43,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/b911028f86854952bd23a2a3717ca043 is 175, key is test_row_0/A:col10/1731689922360/Put/seqid=0 2024-11-15T16:58:43,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742028_1204 (size=48639) 2024-11-15T16:58:44,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689984145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689984145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:44,338 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/b911028f86854952bd23a2a3717ca043 2024-11-15T16:58:44,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/707c7ad4de3649be800ef8073b6ba94d is 50, key is test_row_0/B:col10/1731689922360/Put/seqid=0 2024-11-15T16:58:44,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742029_1205 (size=12301) 2024-11-15T16:58:44,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37314 deadline: 1731689984656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:44,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37294 deadline: 1731689984656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:44,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:44,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:58:44,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/707c7ad4de3649be800ef8073b6ba94d 2024-11-15T16:58:44,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/2628634aaa404b1189e85c5f23580fe7 is 50, key is test_row_0/C:col10/1731689922360/Put/seqid=0 2024-11-15T16:58:44,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742030_1206 (size=12301) 2024-11-15T16:58:44,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/2628634aaa404b1189e85c5f23580fe7 2024-11-15T16:58:44,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/b911028f86854952bd23a2a3717ca043 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/b911028f86854952bd23a2a3717ca043 2024-11-15T16:58:44,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/b911028f86854952bd23a2a3717ca043, entries=250, sequenceid=333, filesize=47.5 K 2024-11-15T16:58:44,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/707c7ad4de3649be800ef8073b6ba94d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/707c7ad4de3649be800ef8073b6ba94d 2024-11-15T16:58:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/707c7ad4de3649be800ef8073b6ba94d, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:58:44,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/2628634aaa404b1189e85c5f23580fe7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2628634aaa404b1189e85c5f23580fe7 2024-11-15T16:58:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T16:58:44,786 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-15T16:58:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2628634aaa404b1189e85c5f23580fe7, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:58:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T16:58:44,791 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:44,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 5800e906b79c2ba33b2668d41c684c23 in 1285ms, sequenceid=333, compaction requested=true 2024-11-15T16:58:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,792 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:44,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,793 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:44,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:44,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5800e906b79c2ba33b2668d41c684c23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,794 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:44,794 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/A is initiating minor compaction (all files) 2024-11-15T16:58:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,794 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/A in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:44,794 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7bef2fec9de5477a8fba0a1db21bda7f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7ef3bf050bf64291924840558956fcd6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/b911028f86854952bd23a2a3717ca043] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=139.7 K 2024-11-15T16:58:44,794 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:44,795 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7bef2fec9de5477a8fba0a1db21bda7f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7ef3bf050bf64291924840558956fcd6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/b911028f86854952bd23a2a3717ca043] 2024-11-15T16:58:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,796 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bef2fec9de5477a8fba0a1db21bda7f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731689918280 2024-11-15T16:58:44,796 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:44,796 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/B is initiating minor compaction (all files) 2024-11-15T16:58:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,796 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/B in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:44,796 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/06613918a9ad4895ba8167d95baa9dd7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/7b5af78ddbf941108348a6807a17d9b4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/bf2e15a92a6f4288a1a0d855e0c79714, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/707c7ad4de3649be800ef8073b6ba94d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=48.7 K 2024-11-15T16:58:44,797 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a4d3c588c74e6c9a6a0ab8bfa96c4f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731689919511 2024-11-15T16:58:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,797 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 06613918a9ad4895ba8167d95baa9dd7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731689918280 2024-11-15T16:58:44,797 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef3bf050bf64291924840558956fcd6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731689920158 2024-11-15T16:58:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,797 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b5af78ddbf941108348a6807a17d9b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731689919511 2024-11-15T16:58:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,798 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2e15a92a6f4288a1a0d855e0c79714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731689920158 2024-11-15T16:58:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,798 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b911028f86854952bd23a2a3717ca043, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689922360 2024-11-15T16:58:44,799 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 707c7ad4de3649be800ef8073b6ba94d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689922360 2024-11-15T16:58:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,809 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,812 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411151cfc2db6aa404b94bbea3c302938d4d0_5800e906b79c2ba33b2668d41c684c23 store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,813 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#B#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,814 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/db77da02c9bb43a0aeaf0ba1ae296478 is 50, key is test_row_0/B:col10/1731689922360/Put/seqid=0 2024-11-15T16:58:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,814 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411151cfc2db6aa404b94bbea3c302938d4d0_5800e906b79c2ba33b2668d41c684c23, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,815 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151cfc2db6aa404b94bbea3c302938d4d0_5800e906b79c2ba33b2668d41c684c23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742031_1207 (size=13085) 2024-11-15T16:58:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,856 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/db77da02c9bb43a0aeaf0ba1ae296478 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db77da02c9bb43a0aeaf0ba1ae296478 2024-11-15T16:58:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,867 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/B of 5800e906b79c2ba33b2668d41c684c23 into db77da02c9bb43a0aeaf0ba1ae296478(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:44,867 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:44,867 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/B, priority=12, startTime=1731689924793; duration=0sec 2024-11-15T16:58:44,867 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:44,867 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:B 2024-11-15T16:58:44,867 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,869 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:44,869 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5800e906b79c2ba33b2668d41c684c23/C is initiating minor compaction (all files) 2024-11-15T16:58:44,869 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5800e906b79c2ba33b2668d41c684c23/C in TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,869 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b39f5d484afb4b63b958e1253b814167, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/00030984a2d940d0b819cd9016189580, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/fed9fec55c544fa2b2bfd51b96d736d6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2628634aaa404b1189e85c5f23580fe7] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp, totalSize=48.7 K 2024-11-15T16:58:44,870 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting b39f5d484afb4b63b958e1253b814167, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731689918280 2024-11-15T16:58:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,871 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 00030984a2d940d0b819cd9016189580, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731689919511 2024-11-15T16:58:44,872 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting fed9fec55c544fa2b2bfd51b96d736d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731689920158 2024-11-15T16:58:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,875 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2628634aaa404b1189e85c5f23580fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689922360 2024-11-15T16:58:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742032_1208 (size=4469) 2024-11-15T16:58:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T16:58:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,899 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,899 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/8f0073ca8a9a49f485f1ec779c4a958e is 50, key is test_row_0/C:col10/1731689922360/Put/seqid=0 2024-11-15T16:58:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742033_1209 (size=13085) 2024-11-15T16:58:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T16:58:44,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,947 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T16:58:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152458301f0edb4973bafb3730a5f0890c_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_1/A:col10/1731689923517/Put/seqid=0 2024-11-15T16:58:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,975 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-15T16:58:44,975 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:44,975 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:44,975 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:44,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,977 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-15T16:58:44,977 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:44,977 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:44,977 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:44,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742034_1210 (size=9914) 2024-11-15T16:58:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,997 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152458301f0edb4973bafb3730a5f0890c_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152458301f0edb4973bafb3730a5f0890c_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:45,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/113ab22b4c394803a04cfe9160ab463b, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/113ab22b4c394803a04cfe9160ab463b is 175, key is test_row_1/A:col10/1731689923517/Put/seqid=0 2024-11-15T16:58:45,004 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-15T16:58:45,004 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:45,005 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,005 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,005 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-15T16:58:45,005 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:45,005 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,005 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:45,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:45,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742035_1211 (size=22561) 2024-11-15T16:58:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T16:58:45,287 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5800e906b79c2ba33b2668d41c684c23#A#compaction#169 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:45,288 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/39dd94d8487f45ed84d91eefa8d17f49 is 175, key is test_row_0/A:col10/1731689922360/Put/seqid=0 2024-11-15T16:58:45,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742036_1212 (size=32039) 2024-11-15T16:58:45,321 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/8f0073ca8a9a49f485f1ec779c4a958e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8f0073ca8a9a49f485f1ec779c4a958e 2024-11-15T16:58:45,325 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/C of 5800e906b79c2ba33b2668d41c684c23 into 8f0073ca8a9a49f485f1ec779c4a958e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:45,326 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:45,326 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/C, priority=12, startTime=1731689924794; duration=0sec 2024-11-15T16:58:45,326 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:45,326 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:C 2024-11-15T16:58:45,409 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/113ab22b4c394803a04cfe9160ab463b 2024-11-15T16:58:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T16:58:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/3e1e328899a446f7b94c1a8a5d8a317a is 50, key is test_row_1/B:col10/1731689923517/Put/seqid=0 2024-11-15T16:58:45,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742037_1213 (size=9857) 2024-11-15T16:58:45,667 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-15T16:58:45,667 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-15T16:58:45,667 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:45,667 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:45,668 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,668 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,668 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,668 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:45,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:45,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:45,697 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/39dd94d8487f45ed84d91eefa8d17f49 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/39dd94d8487f45ed84d91eefa8d17f49 2024-11-15T16:58:45,702 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5800e906b79c2ba33b2668d41c684c23/A of 5800e906b79c2ba33b2668d41c684c23 into 39dd94d8487f45ed84d91eefa8d17f49(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:45,702 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:45,702 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., storeName=5800e906b79c2ba33b2668d41c684c23/A, priority=12, startTime=1731689924792; duration=0sec 2024-11-15T16:58:45,702 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:45,702 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5800e906b79c2ba33b2668d41c684c23:A 2024-11-15T16:58:45,821 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/3e1e328899a446f7b94c1a8a5d8a317a 2024-11-15T16:58:45,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/0f46b98e41884105888ba203de030045 is 50, key is test_row_1/C:col10/1731689923517/Put/seqid=0 2024-11-15T16:58:45,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742038_1214 (size=9857) 2024-11-15T16:58:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T16:58:46,232 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/0f46b98e41884105888ba203de030045 2024-11-15T16:58:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/113ab22b4c394803a04cfe9160ab463b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/113ab22b4c394803a04cfe9160ab463b 2024-11-15T16:58:46,241 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/113ab22b4c394803a04cfe9160ab463b, entries=100, sequenceid=342, filesize=22.0 K 2024-11-15T16:58:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/3e1e328899a446f7b94c1a8a5d8a317a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/3e1e328899a446f7b94c1a8a5d8a317a 2024-11-15T16:58:46,245 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/3e1e328899a446f7b94c1a8a5d8a317a, entries=100, sequenceid=342, filesize=9.6 K 2024-11-15T16:58:46,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/0f46b98e41884105888ba203de030045 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/0f46b98e41884105888ba203de030045 2024-11-15T16:58:46,249 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/0f46b98e41884105888ba203de030045, entries=100, sequenceid=342, filesize=9.6 K 2024-11-15T16:58:46,250 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=13.42 KB/13740 for 5800e906b79c2ba33b2668d41c684c23 in 1303ms, sequenceid=342, compaction requested=false 2024-11-15T16:58:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 5800e906b79c2ba33b2668d41c684c23: 2024-11-15T16:58:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-15T16:58:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-15T16:58:46,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T16:58:46,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-15T16:58:46,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4660 sec 2024-11-15T16:58:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T16:58:46,936 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:48,408 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-15T16:58:48,408 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:48,408 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,408 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:48,437 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-15T16:58:48,437 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:48,437 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,437 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:48,447 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-15T16:58:48,447 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:58:48,447 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,447 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3585 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3463 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1537 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4611 rows 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1552 2024-11-15T16:58:48,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4653 rows 2024-11-15T16:58:48,447 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:58:48,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:58:48,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:48,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:58:48,448 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T16:58:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T16:58:48,451 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689928451"}]},"ts":"1731689928451"} 2024-11-15T16:58:48,453 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T16:58:48,453 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T16:58:48,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T16:58:48,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, UNASSIGN}] 2024-11-15T16:58:48,455 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, UNASSIGN 2024-11-15T16:58:48,456 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:48,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, UNASSIGN because future has completed 2024-11-15T16:58:48,457 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:58:48,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:58:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T16:58:48,609 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:48,609 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:58:48,609 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 5800e906b79c2ba33b2668d41c684c23, disabling compactions & flushes 2024-11-15T16:58:48,609 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:48,609 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:48,609 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. after waiting 0 ms 2024-11-15T16:58:48,609 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:48,610 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 5800e906b79c2ba33b2668d41c684c23 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T16:58:48,616 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153df20c8f367644889c6052f360140446_5800e906b79c2ba33b2668d41c684c23 is 50, key is test_row_0/A:col10/1731689925666/Put/seqid=0 2024-11-15T16:58:48,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742039_1215 (size=12454) 2024-11-15T16:58:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T16:58:49,020 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:49,025 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153df20c8f367644889c6052f360140446_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153df20c8f367644889c6052f360140446_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:49,025 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f1abd18fdae34f4e8c539b2745c1f4a7, store: [table=TestAcidGuarantees family=A region=5800e906b79c2ba33b2668d41c684c23] 2024-11-15T16:58:49,026 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f1abd18fdae34f4e8c539b2745c1f4a7 is 175, key is test_row_0/A:col10/1731689925666/Put/seqid=0 2024-11-15T16:58:49,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742040_1216 (size=31255) 2024-11-15T16:58:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T16:58:49,430 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f1abd18fdae34f4e8c539b2745c1f4a7 2024-11-15T16:58:49,437 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/f7322afba4bf4a9486353c754d7ab037 is 50, key is test_row_0/B:col10/1731689925666/Put/seqid=0 2024-11-15T16:58:49,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742041_1217 (size=12301) 2024-11-15T16:58:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T16:58:49,841 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/f7322afba4bf4a9486353c754d7ab037 2024-11-15T16:58:49,848 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/380d355c53444477a80f82a3151cbb49 is 50, key is test_row_0/C:col10/1731689925666/Put/seqid=0 2024-11-15T16:58:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742042_1218 (size=12301) 2024-11-15T16:58:50,252 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/380d355c53444477a80f82a3151cbb49 2024-11-15T16:58:50,257 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/A/f1abd18fdae34f4e8c539b2745c1f4a7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f1abd18fdae34f4e8c539b2745c1f4a7 2024-11-15T16:58:50,261 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f1abd18fdae34f4e8c539b2745c1f4a7, entries=150, sequenceid=352, filesize=30.5 K 2024-11-15T16:58:50,262 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/B/f7322afba4bf4a9486353c754d7ab037 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/f7322afba4bf4a9486353c754d7ab037 2024-11-15T16:58:50,265 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/f7322afba4bf4a9486353c754d7ab037, entries=150, sequenceid=352, filesize=12.0 K 2024-11-15T16:58:50,266 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/.tmp/C/380d355c53444477a80f82a3151cbb49 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/380d355c53444477a80f82a3151cbb49 2024-11-15T16:58:50,269 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/380d355c53444477a80f82a3151cbb49, entries=150, sequenceid=352, filesize=12.0 K 2024-11-15T16:58:50,270 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5800e906b79c2ba33b2668d41c684c23 in 1661ms, sequenceid=352, compaction requested=true 2024-11-15T16:58:50,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f9bb545c5eaf4424998696302510a0c8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d421549dd346463490f1fb85a3c27c3e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/e24f55e032614a67992a701fb9dd16fb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/67c788758be846f1be387c5fac7eb446, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/205b39f1aa0449598bbd77f67fbe0519, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7624c3e3799a4f49a5d55e672f336068, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/228af541486b4940a205500fc044c6a7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/058aba87f9b7420d969ee2ce8e2108ab, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/181644ac886041fa8965b62e848e6d74, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/4bfe1761d16b40c29f1b0d355a283e48, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1d2f5366570646d8a07b39217d3bb85c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/04799e87be054531bd80309b41346b19, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1829b223e1014aff862d09782f78ccc2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/ae5a1e69df46424a8284244e4dae3528, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/50a7cdc103be42d28c4fa08a1daa8188, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d5e9cfff1d93402695aa1e78ea14ea77, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c4bb161053084084b577bbfea3127b70, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/8c3d4d7895324bf1b6c52cb06681b143, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c40726b710574f8fad74047bb9a481c5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7bef2fec9de5477a8fba0a1db21bda7f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7ef3bf050bf64291924840558956fcd6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/b911028f86854952bd23a2a3717ca043] to archive 2024-11-15T16:58:50,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:58:50,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f9bb545c5eaf4424998696302510a0c8 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f9bb545c5eaf4424998696302510a0c8 2024-11-15T16:58:50,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d421549dd346463490f1fb85a3c27c3e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d421549dd346463490f1fb85a3c27c3e 2024-11-15T16:58:50,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/e24f55e032614a67992a701fb9dd16fb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/e24f55e032614a67992a701fb9dd16fb 2024-11-15T16:58:50,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/67c788758be846f1be387c5fac7eb446 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/67c788758be846f1be387c5fac7eb446 2024-11-15T16:58:50,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/205b39f1aa0449598bbd77f67fbe0519 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/205b39f1aa0449598bbd77f67fbe0519 2024-11-15T16:58:50,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7624c3e3799a4f49a5d55e672f336068 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7624c3e3799a4f49a5d55e672f336068 2024-11-15T16:58:50,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/228af541486b4940a205500fc044c6a7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/228af541486b4940a205500fc044c6a7 2024-11-15T16:58:50,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/058aba87f9b7420d969ee2ce8e2108ab to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/058aba87f9b7420d969ee2ce8e2108ab 2024-11-15T16:58:50,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/181644ac886041fa8965b62e848e6d74 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/181644ac886041fa8965b62e848e6d74 2024-11-15T16:58:50,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/4bfe1761d16b40c29f1b0d355a283e48 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/4bfe1761d16b40c29f1b0d355a283e48 2024-11-15T16:58:50,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1d2f5366570646d8a07b39217d3bb85c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1d2f5366570646d8a07b39217d3bb85c 2024-11-15T16:58:50,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/04799e87be054531bd80309b41346b19 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/04799e87be054531bd80309b41346b19 2024-11-15T16:58:50,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1829b223e1014aff862d09782f78ccc2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/1829b223e1014aff862d09782f78ccc2 2024-11-15T16:58:50,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/ae5a1e69df46424a8284244e4dae3528 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/ae5a1e69df46424a8284244e4dae3528 2024-11-15T16:58:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/50a7cdc103be42d28c4fa08a1daa8188 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/50a7cdc103be42d28c4fa08a1daa8188 2024-11-15T16:58:50,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d5e9cfff1d93402695aa1e78ea14ea77 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/d5e9cfff1d93402695aa1e78ea14ea77 2024-11-15T16:58:50,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c4bb161053084084b577bbfea3127b70 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c4bb161053084084b577bbfea3127b70 2024-11-15T16:58:50,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/8c3d4d7895324bf1b6c52cb06681b143 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/8c3d4d7895324bf1b6c52cb06681b143 2024-11-15T16:58:50,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c40726b710574f8fad74047bb9a481c5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/c40726b710574f8fad74047bb9a481c5 2024-11-15T16:58:50,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7bef2fec9de5477a8fba0a1db21bda7f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7bef2fec9de5477a8fba0a1db21bda7f 2024-11-15T16:58:50,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/21a4d3c588c74e6c9a6a0ab8bfa96c4f 2024-11-15T16:58:50,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7ef3bf050bf64291924840558956fcd6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/7ef3bf050bf64291924840558956fcd6 2024-11-15T16:58:50,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/b911028f86854952bd23a2a3717ca043 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/b911028f86854952bd23a2a3717ca043 2024-11-15T16:58:50,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/12672d8f888b4ce5924e4e5827532482, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1566f598efb64bbcbc96d54b6390bf27, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/a1c74c8e2f6347c785a69694f37472c0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/d3fd803964c14ace8938b6ee046432f7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/108bf8cf27ed4c288a280279089fa4f0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/9c0d4081fb6f4be9aae38bbb55746a7f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/4c041859fa7e40e3b8a58fceace1130c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/875ba3fc20e34b319b2ced1afa00b137, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2709811a3c4845ba8fc0d7972d9cca39, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/405b6b1ec19c45359d001f54d850492e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/eb883a6f33a6452eb07a43bd165ab1ac, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db453afb9dff4c35bfda66c1357e0a91, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2cfcc5746ee24f0f8360ab8777c33dce, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/e29e96daab0c45a6aab0d8f896cffc2f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/07c6376a66d64dfb9a24148120e3e210, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1c2074e07ad84332a91e2f2cf9d06dcc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/64ad38fe15d8426aa7fe1b7e51911650, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/959cde74cccd451aa614563683198f20, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/06613918a9ad4895ba8167d95baa9dd7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/873884c633f0468f805bff454d8bd892, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/7b5af78ddbf941108348a6807a17d9b4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/bf2e15a92a6f4288a1a0d855e0c79714, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/707c7ad4de3649be800ef8073b6ba94d] to archive 2024-11-15T16:58:50,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:58:50,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/12672d8f888b4ce5924e4e5827532482 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/12672d8f888b4ce5924e4e5827532482 2024-11-15T16:58:50,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1566f598efb64bbcbc96d54b6390bf27 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1566f598efb64bbcbc96d54b6390bf27 2024-11-15T16:58:50,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/a1c74c8e2f6347c785a69694f37472c0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/a1c74c8e2f6347c785a69694f37472c0 2024-11-15T16:58:50,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/d3fd803964c14ace8938b6ee046432f7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/d3fd803964c14ace8938b6ee046432f7 2024-11-15T16:58:50,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/108bf8cf27ed4c288a280279089fa4f0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/108bf8cf27ed4c288a280279089fa4f0 2024-11-15T16:58:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/9c0d4081fb6f4be9aae38bbb55746a7f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/9c0d4081fb6f4be9aae38bbb55746a7f 2024-11-15T16:58:50,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/4c041859fa7e40e3b8a58fceace1130c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/4c041859fa7e40e3b8a58fceace1130c 2024-11-15T16:58:50,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/875ba3fc20e34b319b2ced1afa00b137 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/875ba3fc20e34b319b2ced1afa00b137 2024-11-15T16:58:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2709811a3c4845ba8fc0d7972d9cca39 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2709811a3c4845ba8fc0d7972d9cca39 2024-11-15T16:58:50,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/405b6b1ec19c45359d001f54d850492e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/405b6b1ec19c45359d001f54d850492e 2024-11-15T16:58:50,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/eb883a6f33a6452eb07a43bd165ab1ac to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/eb883a6f33a6452eb07a43bd165ab1ac 2024-11-15T16:58:50,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db453afb9dff4c35bfda66c1357e0a91 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db453afb9dff4c35bfda66c1357e0a91 2024-11-15T16:58:50,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2cfcc5746ee24f0f8360ab8777c33dce to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/2cfcc5746ee24f0f8360ab8777c33dce 2024-11-15T16:58:50,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/e29e96daab0c45a6aab0d8f896cffc2f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/e29e96daab0c45a6aab0d8f896cffc2f 2024-11-15T16:58:50,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/07c6376a66d64dfb9a24148120e3e210 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/07c6376a66d64dfb9a24148120e3e210 2024-11-15T16:58:50,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1c2074e07ad84332a91e2f2cf9d06dcc to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/1c2074e07ad84332a91e2f2cf9d06dcc 2024-11-15T16:58:50,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/64ad38fe15d8426aa7fe1b7e51911650 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/64ad38fe15d8426aa7fe1b7e51911650 2024-11-15T16:58:50,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/959cde74cccd451aa614563683198f20 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/959cde74cccd451aa614563683198f20 2024-11-15T16:58:50,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/06613918a9ad4895ba8167d95baa9dd7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/06613918a9ad4895ba8167d95baa9dd7 2024-11-15T16:58:50,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/873884c633f0468f805bff454d8bd892 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/873884c633f0468f805bff454d8bd892 2024-11-15T16:58:50,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/7b5af78ddbf941108348a6807a17d9b4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/7b5af78ddbf941108348a6807a17d9b4 2024-11-15T16:58:50,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/bf2e15a92a6f4288a1a0d855e0c79714 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/bf2e15a92a6f4288a1a0d855e0c79714 2024-11-15T16:58:50,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/707c7ad4de3649be800ef8073b6ba94d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/707c7ad4de3649be800ef8073b6ba94d 2024-11-15T16:58:50,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48b1413cf92740788a66eeb4ca621179, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/e2186002e497421a9a40c1f9734c6f7a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/7a081a7b0e984a598c205c7d91467978, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/074b3858de4f4c1eb3372db0b2b7402e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c9492396d14e4a7f8670c6b2ccc69f07, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/1e5e50b93d7c452aa3dd1d9979ef190f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48adf1dc84384fecb0ffa81d837d3ddb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/10806aff6dd0426aae6c20cf056d6723, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/cf7ca53a740d449e93e7847c09c3c9bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/93dbebcf4bab4584b26fcfd523ef3e92, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b8c92a2b5ee4495cb099335ee0f8f01f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/06a36789a4ad458ea0e3c209625e5660, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/d7c2ae6375db4a3580210acd36d7c2a9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8318bbc194a24561bf705055609b174e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2abfdcaa690f4269b94ffeede5dad400, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c254a164bdc24536854c107b6d0ac748, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/88f46e3473a5417887fedec5c090c589, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/3938dc3232d44e4abd90872dcb081dfe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b39f5d484afb4b63b958e1253b814167, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/aa1251611c5b4a18af2e42d69979b5d5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/00030984a2d940d0b819cd9016189580, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/fed9fec55c544fa2b2bfd51b96d736d6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2628634aaa404b1189e85c5f23580fe7] to archive 2024-11-15T16:58:50,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:58:50,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48b1413cf92740788a66eeb4ca621179 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48b1413cf92740788a66eeb4ca621179 2024-11-15T16:58:50,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/e2186002e497421a9a40c1f9734c6f7a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/e2186002e497421a9a40c1f9734c6f7a 2024-11-15T16:58:50,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/7a081a7b0e984a598c205c7d91467978 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/7a081a7b0e984a598c205c7d91467978 2024-11-15T16:58:50,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/074b3858de4f4c1eb3372db0b2b7402e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/074b3858de4f4c1eb3372db0b2b7402e 2024-11-15T16:58:50,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c9492396d14e4a7f8670c6b2ccc69f07 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c9492396d14e4a7f8670c6b2ccc69f07 2024-11-15T16:58:50,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/1e5e50b93d7c452aa3dd1d9979ef190f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/1e5e50b93d7c452aa3dd1d9979ef190f 2024-11-15T16:58:50,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48adf1dc84384fecb0ffa81d837d3ddb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/48adf1dc84384fecb0ffa81d837d3ddb 2024-11-15T16:58:50,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/10806aff6dd0426aae6c20cf056d6723 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/10806aff6dd0426aae6c20cf056d6723 2024-11-15T16:58:50,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/cf7ca53a740d449e93e7847c09c3c9bb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/cf7ca53a740d449e93e7847c09c3c9bb 2024-11-15T16:58:50,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/93dbebcf4bab4584b26fcfd523ef3e92 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/93dbebcf4bab4584b26fcfd523ef3e92 2024-11-15T16:58:50,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b8c92a2b5ee4495cb099335ee0f8f01f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b8c92a2b5ee4495cb099335ee0f8f01f 2024-11-15T16:58:50,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/06a36789a4ad458ea0e3c209625e5660 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/06a36789a4ad458ea0e3c209625e5660 2024-11-15T16:58:50,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/d7c2ae6375db4a3580210acd36d7c2a9 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/d7c2ae6375db4a3580210acd36d7c2a9 2024-11-15T16:58:50,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8318bbc194a24561bf705055609b174e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8318bbc194a24561bf705055609b174e 2024-11-15T16:58:50,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2abfdcaa690f4269b94ffeede5dad400 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2abfdcaa690f4269b94ffeede5dad400 2024-11-15T16:58:50,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c254a164bdc24536854c107b6d0ac748 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/c254a164bdc24536854c107b6d0ac748 2024-11-15T16:58:50,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/88f46e3473a5417887fedec5c090c589 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/88f46e3473a5417887fedec5c090c589 2024-11-15T16:58:50,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/3938dc3232d44e4abd90872dcb081dfe to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/3938dc3232d44e4abd90872dcb081dfe 2024-11-15T16:58:50,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b39f5d484afb4b63b958e1253b814167 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/b39f5d484afb4b63b958e1253b814167 2024-11-15T16:58:50,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/aa1251611c5b4a18af2e42d69979b5d5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/aa1251611c5b4a18af2e42d69979b5d5 2024-11-15T16:58:50,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/00030984a2d940d0b819cd9016189580 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/00030984a2d940d0b819cd9016189580 2024-11-15T16:58:50,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/fed9fec55c544fa2b2bfd51b96d736d6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/fed9fec55c544fa2b2bfd51b96d736d6 2024-11-15T16:58:50,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2628634aaa404b1189e85c5f23580fe7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/2628634aaa404b1189e85c5f23580fe7 2024-11-15T16:58:50,352 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/recovered.edits/355.seqid, newMaxSeqId=355, maxSeqId=4 2024-11-15T16:58:50,352 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. 2024-11-15T16:58:50,352 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 5800e906b79c2ba33b2668d41c684c23: Waiting for close lock at 1731689928609Running coprocessor pre-close hooks at 1731689928609Disabling compacts and flushes for region at 1731689928609Disabling writes for close at 1731689928609Obtaining lock to block concurrent updates at 1731689928610 (+1 ms)Preparing flush snapshotting stores in 5800e906b79c2ba33b2668d41c684c23 at 1731689928610Finished memstore snapshotting TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731689928610Flushing stores of TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23. at 1731689928610Flushing 5800e906b79c2ba33b2668d41c684c23/A: creating writer at 1731689928610Flushing mob file 5800e906b79c2ba33b2668d41c684c23/A: appending metadata at 1731689928615 (+5 ms)Flushing mob file 5800e906b79c2ba33b2668d41c684c23/A: closing flushed file at 1731689928615Flushing 5800e906b79c2ba33b2668d41c684c23/A: appending metadata at 1731689929026 (+411 ms)Flushing 5800e906b79c2ba33b2668d41c684c23/A: closing flushed file at 1731689929026Flushing 5800e906b79c2ba33b2668d41c684c23/B: creating writer at 1731689929434 (+408 ms)Flushing 5800e906b79c2ba33b2668d41c684c23/B: appending metadata at 1731689929437 (+3 ms)Flushing 5800e906b79c2ba33b2668d41c684c23/B: closing flushed file at 1731689929437Flushing 5800e906b79c2ba33b2668d41c684c23/C: creating writer at 1731689929845 (+408 ms)Flushing 5800e906b79c2ba33b2668d41c684c23/C: appending metadata at 1731689929848 (+3 ms)Flushing 5800e906b79c2ba33b2668d41c684c23/C: closing flushed file at 1731689929848Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@438a69a9: reopening flushed file at 1731689930256 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18383dc6: reopening flushed file at 1731689930261 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28546dbe: reopening flushed file at 1731689930265 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5800e906b79c2ba33b2668d41c684c23 in 1661ms, sequenceid=352, compaction requested=true at 1731689930270 (+5 ms)Writing region close event to WAL at 1731689930349 (+79 ms)Running coprocessor post-close hooks at 1731689930352 (+3 ms)Closed at 1731689930352 2024-11-15T16:58:50,354 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,355 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5800e906b79c2ba33b2668d41c684c23, regionState=CLOSED 2024-11-15T16:58:50,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:58:50,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T16:58:50,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 5800e906b79c2ba33b2668d41c684c23, server=a9bde5b3a1b5,35119,1731689872090 in 1.9000 sec 2024-11-15T16:58:50,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-15T16:58:50,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5800e906b79c2ba33b2668d41c684c23, UNASSIGN in 1.9040 sec 2024-11-15T16:58:50,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T16:58:50,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9070 sec 2024-11-15T16:58:50,363 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689930363"}]},"ts":"1731689930363"} 2024-11-15T16:58:50,364 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T16:58:50,364 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T16:58:50,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-15T16:58:50,498 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T16:58:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T16:58:50,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:50,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T16:58:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:50,608 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T16:58:50,609 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:50,612 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,615 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/recovered.edits] 2024-11-15T16:58:50,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/113ab22b4c394803a04cfe9160ab463b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/113ab22b4c394803a04cfe9160ab463b 2024-11-15T16:58:50,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/39dd94d8487f45ed84d91eefa8d17f49 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/39dd94d8487f45ed84d91eefa8d17f49 2024-11-15T16:58:50,619 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f1abd18fdae34f4e8c539b2745c1f4a7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/A/f1abd18fdae34f4e8c539b2745c1f4a7 2024-11-15T16:58:50,622 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/3e1e328899a446f7b94c1a8a5d8a317a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/3e1e328899a446f7b94c1a8a5d8a317a 2024-11-15T16:58:50,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db77da02c9bb43a0aeaf0ba1ae296478 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/db77da02c9bb43a0aeaf0ba1ae296478 2024-11-15T16:58:50,624 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/f7322afba4bf4a9486353c754d7ab037 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/B/f7322afba4bf4a9486353c754d7ab037 2024-11-15T16:58:50,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/0f46b98e41884105888ba203de030045 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/0f46b98e41884105888ba203de030045 2024-11-15T16:58:50,627 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/380d355c53444477a80f82a3151cbb49 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/380d355c53444477a80f82a3151cbb49 2024-11-15T16:58:50,628 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8f0073ca8a9a49f485f1ec779c4a958e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/C/8f0073ca8a9a49f485f1ec779c4a958e 2024-11-15T16:58:50,630 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/recovered.edits/355.seqid to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23/recovered.edits/355.seqid 2024-11-15T16:58:50,631 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,631 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T16:58:50,631 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T16:58:50,632 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T16:58:50,636 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150928cace83114559a55fe6c6f135f5fa_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150928cace83114559a55fe6c6f135f5fa_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,637 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151578df033afb49bbbba9bd48b7f9fa07_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151578df033afb49bbbba9bd48b7f9fa07_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,638 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151d78c20fc4944f05998d1d94d5469087_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151d78c20fc4944f05998d1d94d5469087_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,640 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151e3822d5193145438f6b04b23e8f2cdb_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151e3822d5193145438f6b04b23e8f2cdb_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,641 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115228adcd2765b4659ae0e9b413dc00cc2_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115228adcd2765b4659ae0e9b413dc00cc2_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,642 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152458301f0edb4973bafb3730a5f0890c_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152458301f0edb4973bafb3730a5f0890c_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,643 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153df20c8f367644889c6052f360140446_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153df20c8f367644889c6052f360140446_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,645 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111557653ab9311040c3a9a0f3002ee94d9a_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111557653ab9311040c3a9a0f3002ee94d9a_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,646 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155da7b470cb20463f93a33334d4383507_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155da7b470cb20463f93a33334d4383507_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,647 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560673b99b2ab469688d1f99603399bfc_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560673b99b2ab469688d1f99603399bfc_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,649 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115632b6a086e76479dbd71ecfef087cdb9_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115632b6a086e76479dbd71ecfef087cdb9_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,650 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115767c1134863d4d2fbd1b47767990c600_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115767c1134863d4d2fbd1b47767990c600_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,652 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579627a36f0a845ef88274945e8a933c0_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579627a36f0a845ef88274945e8a933c0_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,655 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159648690bc163443ebaa8d317d85957e7_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159648690bc163443ebaa8d317d85957e7_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,656 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115977d86d7f6be495780391cf7369b3e47_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115977d86d7f6be495780391cf7369b3e47_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,658 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b945f7e30f14d07898b31c78d5cc844_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b945f7e30f14d07898b31c78d5cc844_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,660 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4bb098b3ad24503933f46f0117aa4a2_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4bb098b3ad24503933f46f0117aa4a2_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,661 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d178dfd5da6742cd941e0cbafefd0162_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d178dfd5da6742cd941e0cbafefd0162_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,662 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d609f2a3f86a41dd850664a5101af276_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d609f2a3f86a41dd850664a5101af276_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,663 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f518a05ad5fb4190a833472c676de433_5800e906b79c2ba33b2668d41c684c23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f518a05ad5fb4190a833472c676de433_5800e906b79c2ba33b2668d41c684c23 2024-11-15T16:58:50,664 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T16:58:50,666 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:50,669 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T16:58:50,671 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T16:58:50,672 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:50,672 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T16:58:50,672 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731689930672"}]},"ts":"9223372036854775807"} 2024-11-15T16:58:50,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T16:58:50,674 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5800e906b79c2ba33b2668d41c684c23, NAME => 'TestAcidGuarantees,,1731689903139.5800e906b79c2ba33b2668d41c684c23.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T16:58:50,674 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T16:58:50,674 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731689930674"}]},"ts":"9223372036854775807"} 2024-11-15T16:58:50,676 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T16:58:50,676 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:50,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-15T16:58:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T16:58:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T16:58:50,716 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:50,725 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240 (was 239) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-915587138_22 at /127.0.0.1:51834 [Waiting for operation #1202] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1850901185_22 at /127.0.0.1:41348 [Waiting for operation #1021] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 339) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5517 (was 5869) 2024-11-15T16:58:50,734 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=5517 2024-11-15T16:58:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T16:58:50,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:58:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T16:58:50,738 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:58:50,738 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:50,738 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-15T16:58:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T16:58:50,738 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:58:50,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742043_1219 (size=959) 2024-11-15T16:58:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T16:58:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T16:58:51,146 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T16:58:51,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742044_1220 (size=53) 2024-11-15T16:58:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T16:58:51,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:58:51,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e122d8e8e1d3a6c89c4dc261c769f99f, disabling compactions & flushes 2024-11-15T16:58:51,552 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:51,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:51,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. after waiting 0 ms 2024-11-15T16:58:51,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:51,552 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:51,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e122d8e8e1d3a6c89c4dc261c769f99f: Waiting for close lock at 1731689931552Disabling compacts and flushes for region at 1731689931552Disabling writes for close at 1731689931552Writing region close event to WAL at 1731689931552Closed at 1731689931552 2024-11-15T16:58:51,554 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:58:51,554 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731689931554"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689931554"}]},"ts":"1731689931554"} 2024-11-15T16:58:51,555 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:58:51,556 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:58:51,556 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689931556"}]},"ts":"1731689931556"} 2024-11-15T16:58:51,557 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T16:58:51,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, ASSIGN}] 2024-11-15T16:58:51,558 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, ASSIGN 2024-11-15T16:58:51,559 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=false 2024-11-15T16:58:51,710 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=e122d8e8e1d3a6c89c4dc261c769f99f, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:51,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, ASSIGN because future has completed 2024-11-15T16:58:51,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:58:51,816 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T16:58:51,867 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:51,867 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:58:51,867 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,867 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:58:51,867 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,867 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,868 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,870 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e122d8e8e1d3a6c89c4dc261c769f99f columnFamilyName A 2024-11-15T16:58:51,870 DEBUG [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:51,870 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] regionserver.HStore(327): Store=e122d8e8e1d3a6c89c4dc261c769f99f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:51,870 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,871 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e122d8e8e1d3a6c89c4dc261c769f99f columnFamilyName B 2024-11-15T16:58:51,871 DEBUG [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:51,872 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] regionserver.HStore(327): Store=e122d8e8e1d3a6c89c4dc261c769f99f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:51,872 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,873 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e122d8e8e1d3a6c89c4dc261c769f99f columnFamilyName C 2024-11-15T16:58:51,873 DEBUG [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:58:51,873 INFO [StoreOpener-e122d8e8e1d3a6c89c4dc261c769f99f-1 {}] regionserver.HStore(327): Store=e122d8e8e1d3a6c89c4dc261c769f99f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:58:51,873 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,874 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,874 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,875 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,875 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,875 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:58:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T16:58:51,876 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,878 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:58:51,878 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened e122d8e8e1d3a6c89c4dc261c769f99f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71237177, jitterRate=0.0615166574716568}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:58:51,878 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:51,879 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for e122d8e8e1d3a6c89c4dc261c769f99f: Running coprocessor pre-open hook at 1731689931867Writing region info on filesystem at 1731689931867Initializing all the Stores at 1731689931868 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689931868Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689931868Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689931868Cleaning up temporary data from old regions at 1731689931875 (+7 ms)Running coprocessor post-open hooks at 1731689931878 (+3 ms)Region opened successfully at 1731689931879 (+1 ms) 2024-11-15T16:58:51,880 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., pid=67, masterSystemTime=1731689931864 2024-11-15T16:58:51,881 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:51,881 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:51,882 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=e122d8e8e1d3a6c89c4dc261c769f99f, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:51,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:58:51,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T16:58:51,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 in 172 msec 2024-11-15T16:58:51,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-15T16:58:51,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, ASSIGN in 328 msec 2024-11-15T16:58:51,887 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:58:51,888 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689931887"}]},"ts":"1731689931887"} 2024-11-15T16:58:51,889 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T16:58:51,889 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:58:51,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-15T16:58:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T16:58:52,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:52,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,889 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0bb69a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,890 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f49276a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,894 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:58:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:58:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fc03a96 2024-11-15T16:58:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:58:52,895 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:58:52,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,897 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4809e84e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77439d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42251a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3289eb1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,903 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52cb674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b195de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,906 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc08266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,907 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53668780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc1c0af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,910 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f65362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@278cbac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f61e3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7041f94c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b12f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d764711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a4fc22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,919 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d441e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f8b949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:58:52,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:58:52,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:58:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:58:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:58:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eee00f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:58:52,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:58:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:58:52,923 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:58:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e909d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:58:52,924 DEBUG [Thread-1040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,924 DEBUG [Thread-1055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,924 DEBUG [Thread-1044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,924 DEBUG [Thread-1038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,924 DEBUG [Thread-1049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,924 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:52,925 DEBUG [Thread-1053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,926 DEBUG [Thread-1036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,926 DEBUG [Thread-1051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,926 DEBUG [Thread-1042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:58:52,926 DEBUG [Thread-1044 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,927 DEBUG [Thread-1044 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,927 DEBUG [Thread-1042 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,927 DEBUG [Thread-1042 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:58:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:58:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T16:58:52,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T16:58:52,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:52,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:58:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:58:52,934 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:52,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:52,939 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:52,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:58:52,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/9812f21204064146a1a86798c8ae68a3 is 50, key is test_row_0/A:col10/1731689932938/Put/seqid=0 2024-11-15T16:58:52,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742045_1221 (size=12001) 2024-11-15T16:58:52,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689992992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689992996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689992998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689992999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689992999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T16:58:53,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T16:58:53,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689993106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689993106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689993106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689993115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689993116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T16:58:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T16:58:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689993315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689993315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689993315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689993326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689993326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/9812f21204064146a1a86798c8ae68a3 2024-11-15T16:58:53,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T16:58:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/bd15ebca5641476e970d2814ad8eaa45 is 50, key is test_row_0/B:col10/1731689932938/Put/seqid=0 2024-11-15T16:58:53,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742046_1222 (size=12001) 2024-11-15T16:58:53,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/bd15ebca5641476e970d2814ad8eaa45 2024-11-15T16:58:53,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/17fc4c0b849d42e2949bbeb7370377bb is 50, key is test_row_0/C:col10/1731689932938/Put/seqid=0 2024-11-15T16:58:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742047_1223 (size=12001) 2024-11-15T16:58:53,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T16:58:53,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:53,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T16:58:53,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689993625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689993625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689993625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689993635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689993636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:53,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T16:58:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/17fc4c0b849d42e2949bbeb7370377bb 2024-11-15T16:58:53,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/9812f21204064146a1a86798c8ae68a3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9812f21204064146a1a86798c8ae68a3 2024-11-15T16:58:53,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9812f21204064146a1a86798c8ae68a3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T16:58:53,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/bd15ebca5641476e970d2814ad8eaa45 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/bd15ebca5641476e970d2814ad8eaa45 2024-11-15T16:58:53,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/bd15ebca5641476e970d2814ad8eaa45, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T16:58:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/17fc4c0b849d42e2949bbeb7370377bb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/17fc4c0b849d42e2949bbeb7370377bb 2024-11-15T16:58:53,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T16:58:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:53,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:53,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/17fc4c0b849d42e2949bbeb7370377bb, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T16:58:53,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e122d8e8e1d3a6c89c4dc261c769f99f in 917ms, sequenceid=13, compaction requested=false 2024-11-15T16:58:53,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:54,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T16:58:54,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:54,007 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:58:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/a872eefb4c7a4645abbc8c0f7444beeb is 50, key is test_row_0/A:col10/1731689932998/Put/seqid=0 2024-11-15T16:58:54,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742048_1224 (size=12001) 2024-11-15T16:58:54,030 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/a872eefb4c7a4645abbc8c0f7444beeb 2024-11-15T16:58:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6a7d048dd1aa48f38b7539623ce90519 is 50, key is test_row_0/B:col10/1731689932998/Put/seqid=0 2024-11-15T16:58:54,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742049_1225 (size=12001) 2024-11-15T16:58:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T16:58:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:54,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:54,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689994146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689994146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689994150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689994152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689994152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689994265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689994265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689994265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,446 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6a7d048dd1aa48f38b7539623ce90519 2024-11-15T16:58:54,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/bf8534ce5dcb4829bd244d7b1543756e is 50, key is test_row_0/C:col10/1731689932998/Put/seqid=0 2024-11-15T16:58:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742050_1226 (size=12001) 2024-11-15T16:58:54,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689994476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689994476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689994476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689994785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689994786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689994786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:54,859 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/bf8534ce5dcb4829bd244d7b1543756e 2024-11-15T16:58:54,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/a872eefb4c7a4645abbc8c0f7444beeb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/a872eefb4c7a4645abbc8c0f7444beeb 2024-11-15T16:58:54,870 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/a872eefb4c7a4645abbc8c0f7444beeb, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T16:58:54,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6a7d048dd1aa48f38b7539623ce90519 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6a7d048dd1aa48f38b7539623ce90519 2024-11-15T16:58:54,875 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6a7d048dd1aa48f38b7539623ce90519, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T16:58:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/bf8534ce5dcb4829bd244d7b1543756e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/bf8534ce5dcb4829bd244d7b1543756e 2024-11-15T16:58:54,880 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/bf8534ce5dcb4829bd244d7b1543756e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T16:58:54,881 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for e122d8e8e1d3a6c89c4dc261c769f99f in 875ms, sequenceid=38, compaction requested=false 2024-11-15T16:58:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T16:58:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T16:58:54,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T16:58:54,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9490 sec 2024-11-15T16:58:54,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9610 sec 2024-11-15T16:58:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T16:58:55,076 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:55,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T16:58:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T16:58:55,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:55,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:55,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:55,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:55,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T16:58:55,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/cb22e4eae9c54b20a11654b4057b757f is 50, key is test_row_0/A:col10/1731689935166/Put/seqid=0 2024-11-15T16:58:55,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742051_1227 (size=14341) 2024-11-15T16:58:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T16:58:55,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689995211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689995212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T16:58:55,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:55,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689995295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689995295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689995296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689995326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689995326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T16:58:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T16:58:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689995536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689995536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T16:58:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/cb22e4eae9c54b20a11654b4057b757f 2024-11-15T16:58:55,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/f6b6994de6c247bcaeb6624e7b698494 is 50, key is test_row_0/B:col10/1731689935166/Put/seqid=0 2024-11-15T16:58:55,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742052_1228 (size=12001) 2024-11-15T16:58:55,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T16:58:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T16:58:55,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T16:58:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:55,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689995845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689995845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:55,875 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:58:55,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/f6b6994de6c247bcaeb6624e7b698494 2024-11-15T16:58:56,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/e52577a538e84a9781e6df5f9218719a is 50, key is test_row_0/C:col10/1731689935166/Put/seqid=0 2024-11-15T16:58:56,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T16:58:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:56,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742053_1229 (size=12001) 2024-11-15T16:58:56,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/e52577a538e84a9781e6df5f9218719a 2024-11-15T16:58:56,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/cb22e4eae9c54b20a11654b4057b757f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cb22e4eae9c54b20a11654b4057b757f 2024-11-15T16:58:56,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cb22e4eae9c54b20a11654b4057b757f, entries=200, sequenceid=52, filesize=14.0 K 2024-11-15T16:58:56,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/f6b6994de6c247bcaeb6624e7b698494 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/f6b6994de6c247bcaeb6624e7b698494 2024-11-15T16:58:56,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/f6b6994de6c247bcaeb6624e7b698494, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T16:58:56,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/e52577a538e84a9781e6df5f9218719a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e52577a538e84a9781e6df5f9218719a 2024-11-15T16:58:56,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e52577a538e84a9781e6df5f9218719a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T16:58:56,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e122d8e8e1d3a6c89c4dc261c769f99f in 859ms, sequenceid=52, compaction requested=true 2024-11-15T16:58:56,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:56,028 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:56,028 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:56,029 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:56,029 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:56,029 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:58:56,029 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:58:56,029 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,029 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,029 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9812f21204064146a1a86798c8ae68a3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/a872eefb4c7a4645abbc8c0f7444beeb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cb22e4eae9c54b20a11654b4057b757f] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=37.4 K 2024-11-15T16:58:56,029 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/bd15ebca5641476e970d2814ad8eaa45, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6a7d048dd1aa48f38b7539623ce90519, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/f6b6994de6c247bcaeb6624e7b698494] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=35.2 K 2024-11-15T16:58:56,030 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting bd15ebca5641476e970d2814ad8eaa45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731689932935 2024-11-15T16:58:56,030 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9812f21204064146a1a86798c8ae68a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731689932935 2024-11-15T16:58:56,030 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a7d048dd1aa48f38b7539623ce90519, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689932995 2024-11-15T16:58:56,030 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a872eefb4c7a4645abbc8c0f7444beeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689932995 2024-11-15T16:58:56,030 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f6b6994de6c247bcaeb6624e7b698494, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689934149 2024-11-15T16:58:56,031 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb22e4eae9c54b20a11654b4057b757f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689934143 2024-11-15T16:58:56,045 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:56,046 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/dccd1abbf1a4471ab1294731594c5464 is 50, key is test_row_0/A:col10/1731689935166/Put/seqid=0 2024-11-15T16:58:56,046 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:56,047 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/2d94c79b02c04cc9a675422913b04b01 is 50, key is test_row_0/B:col10/1731689935166/Put/seqid=0 2024-11-15T16:58:56,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742055_1231 (size=12104) 2024-11-15T16:58:56,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742054_1230 (size=12104) 2024-11-15T16:58:56,088 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/dccd1abbf1a4471ab1294731594c5464 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/dccd1abbf1a4471ab1294731594c5464 2024-11-15T16:58:56,094 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into dccd1abbf1a4471ab1294731594c5464(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:56,095 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:56,095 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=13, startTime=1731689936027; duration=0sec 2024-11-15T16:58:56,095 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:56,095 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:58:56,095 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:56,096 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:56,096 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:58:56,096 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,096 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/17fc4c0b849d42e2949bbeb7370377bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/bf8534ce5dcb4829bd244d7b1543756e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e52577a538e84a9781e6df5f9218719a] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=35.2 K 2024-11-15T16:58:56,096 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17fc4c0b849d42e2949bbeb7370377bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731689932935 2024-11-15T16:58:56,097 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8534ce5dcb4829bd244d7b1543756e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689932995 2024-11-15T16:58:56,098 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52577a538e84a9781e6df5f9218719a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689934149 2024-11-15T16:58:56,106 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:56,106 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/cbce6563b58f4d8f8194395de34e1019 is 50, key is test_row_0/C:col10/1731689935166/Put/seqid=0 2024-11-15T16:58:56,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742056_1232 (size=12104) 2024-11-15T16:58:56,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T16:58:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,154 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T16:58:56,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/111838aad37b4537a0008ab4431aad61 is 50, key is test_row_0/A:col10/1731689935201/Put/seqid=0 2024-11-15T16:58:56,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742057_1233 (size=12001) 2024-11-15T16:58:56,176 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/111838aad37b4537a0008ab4431aad61 2024-11-15T16:58:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6bc8b0be7358481eba609cce1530aa3c is 50, key is test_row_0/B:col10/1731689935201/Put/seqid=0 2024-11-15T16:58:56,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742058_1234 (size=12001) 2024-11-15T16:58:56,212 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6bc8b0be7358481eba609cce1530aa3c 2024-11-15T16:58:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T16:58:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/7b092f4892e7486da9f3eaa434ff9568 is 50, key is test_row_0/C:col10/1731689935201/Put/seqid=0 2024-11-15T16:58:56,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742059_1235 (size=12001) 2024-11-15T16:58:56,249 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/7b092f4892e7486da9f3eaa434ff9568 2024-11-15T16:58:56,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/111838aad37b4537a0008ab4431aad61 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/111838aad37b4537a0008ab4431aad61 2024-11-15T16:58:56,263 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/111838aad37b4537a0008ab4431aad61, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T16:58:56,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6bc8b0be7358481eba609cce1530aa3c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6bc8b0be7358481eba609cce1530aa3c 2024-11-15T16:58:56,271 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6bc8b0be7358481eba609cce1530aa3c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T16:58:56,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/7b092f4892e7486da9f3eaa434ff9568 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/7b092f4892e7486da9f3eaa434ff9568 2024-11-15T16:58:56,287 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/7b092f4892e7486da9f3eaa434ff9568, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T16:58:56,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for e122d8e8e1d3a6c89c4dc261c769f99f in 134ms, sequenceid=75, compaction requested=false 2024-11-15T16:58:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T16:58:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T16:58:56,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T16:58:56,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2090 sec 2024-11-15T16:58:56,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2160 sec 2024-11-15T16:58:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:56,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:58:56,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/4aae7bd1d29f4319be95fa80c5b0cf26 is 50, key is test_row_0/A:col10/1731689936314/Put/seqid=0 2024-11-15T16:58:56,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742060_1236 (size=14341) 2024-11-15T16:58:56,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/4aae7bd1d29f4319be95fa80c5b0cf26 2024-11-15T16:58:56,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/823b7b316f9a428f969a765f3be1aa7e is 50, key is test_row_0/B:col10/1731689936314/Put/seqid=0 2024-11-15T16:58:56,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689996360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689996361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689996365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689996367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689996367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742061_1237 (size=12001) 2024-11-15T16:58:56,471 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/2d94c79b02c04cc9a675422913b04b01 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/2d94c79b02c04cc9a675422913b04b01 2024-11-15T16:58:56,477 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into 2d94c79b02c04cc9a675422913b04b01(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:56,477 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:56,477 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=13, startTime=1731689936028; duration=0sec 2024-11-15T16:58:56,477 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:56,477 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:58:56,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689996475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689996476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689996476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689996476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689996476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,548 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/cbce6563b58f4d8f8194395de34e1019 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/cbce6563b58f4d8f8194395de34e1019 2024-11-15T16:58:56,554 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into cbce6563b58f4d8f8194395de34e1019(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:56,554 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:56,554 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=13, startTime=1731689936028; duration=0sec 2024-11-15T16:58:56,554 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:56,554 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:58:56,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689996685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689996685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689996685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689996686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689996686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:56,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/823b7b316f9a428f969a765f3be1aa7e 2024-11-15T16:58:56,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/237a17164453419c8ee5c688f48ae7bb is 50, key is test_row_0/C:col10/1731689936314/Put/seqid=0 2024-11-15T16:58:56,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742062_1238 (size=12001) 2024-11-15T16:58:56,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/237a17164453419c8ee5c688f48ae7bb 2024-11-15T16:58:56,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/4aae7bd1d29f4319be95fa80c5b0cf26 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/4aae7bd1d29f4319be95fa80c5b0cf26 2024-11-15T16:58:56,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/4aae7bd1d29f4319be95fa80c5b0cf26, entries=200, sequenceid=87, filesize=14.0 K 2024-11-15T16:58:56,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/823b7b316f9a428f969a765f3be1aa7e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/823b7b316f9a428f969a765f3be1aa7e 2024-11-15T16:58:56,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/823b7b316f9a428f969a765f3be1aa7e, entries=150, sequenceid=87, filesize=11.7 K 2024-11-15T16:58:56,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/237a17164453419c8ee5c688f48ae7bb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/237a17164453419c8ee5c688f48ae7bb 2024-11-15T16:58:56,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/237a17164453419c8ee5c688f48ae7bb, entries=150, sequenceid=87, filesize=11.7 K 2024-11-15T16:58:56,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e122d8e8e1d3a6c89c4dc261c769f99f in 504ms, sequenceid=87, compaction requested=true 2024-11-15T16:58:56,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:56,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:56,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:56,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:56,822 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:56,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:56,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:56,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:56,822 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:56,823 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:56,823 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:58:56,823 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,823 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/2d94c79b02c04cc9a675422913b04b01, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6bc8b0be7358481eba609cce1530aa3c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/823b7b316f9a428f969a765f3be1aa7e] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=35.3 K 2024-11-15T16:58:56,824 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:56,824 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:58:56,824 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:56,824 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/dccd1abbf1a4471ab1294731594c5464, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/111838aad37b4537a0008ab4431aad61, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/4aae7bd1d29f4319be95fa80c5b0cf26] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=37.5 K 2024-11-15T16:58:56,824 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d94c79b02c04cc9a675422913b04b01, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689934149 2024-11-15T16:58:56,825 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting dccd1abbf1a4471ab1294731594c5464, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689934149 2024-11-15T16:58:56,825 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc8b0be7358481eba609cce1530aa3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731689935201 2024-11-15T16:58:56,825 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 111838aad37b4537a0008ab4431aad61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731689935201 2024-11-15T16:58:56,826 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 823b7b316f9a428f969a765f3be1aa7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731689936314 2024-11-15T16:58:56,826 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aae7bd1d29f4319be95fa80c5b0cf26, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731689936306 2024-11-15T16:58:56,847 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:56,847 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:56,848 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/78cb1caf353b494e95e9e907a948e3bb is 50, key is test_row_0/A:col10/1731689936314/Put/seqid=0 2024-11-15T16:58:56,849 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/cca2c5b9952b40a4a7a70194daf043ee is 50, key is test_row_0/B:col10/1731689936314/Put/seqid=0 2024-11-15T16:58:56,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742063_1239 (size=12207) 2024-11-15T16:58:56,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742064_1240 (size=12207) 2024-11-15T16:58:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:56,998 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:56,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:58:57,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/6f7fa71f0e0442d68db586680cb5ebc6 is 50, key is test_row_0/A:col10/1731689936365/Put/seqid=0 2024-11-15T16:58:57,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689997005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742065_1241 (size=12001) 2024-11-15T16:58:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689997006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/6f7fa71f0e0442d68db586680cb5ebc6 2024-11-15T16:58:57,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689997007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689997008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689997009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/b85b2e433d8e499ebe0760df59f4c5d3 is 50, key is test_row_0/B:col10/1731689936365/Put/seqid=0 2024-11-15T16:58:57,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742066_1242 (size=12001) 2024-11-15T16:58:57,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689997116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689997116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689997116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689997126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689997126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T16:58:57,226 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:57,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T16:58:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T16:58:57,242 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:57,243 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:57,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:57,261 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/78cb1caf353b494e95e9e907a948e3bb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/78cb1caf353b494e95e9e907a948e3bb 2024-11-15T16:58:57,263 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/cca2c5b9952b40a4a7a70194daf043ee as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cca2c5b9952b40a4a7a70194daf043ee 2024-11-15T16:58:57,266 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 78cb1caf353b494e95e9e907a948e3bb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:57,266 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:57,266 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=13, startTime=1731689936821; duration=0sec 2024-11-15T16:58:57,266 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:57,266 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:58:57,266 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:58:57,268 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:58:57,268 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:58:57,268 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:57,268 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/cbce6563b58f4d8f8194395de34e1019, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/7b092f4892e7486da9f3eaa434ff9568, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/237a17164453419c8ee5c688f48ae7bb] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=35.3 K 2024-11-15T16:58:57,268 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbce6563b58f4d8f8194395de34e1019, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731689934149 2024-11-15T16:58:57,269 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b092f4892e7486da9f3eaa434ff9568, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731689935201 2024-11-15T16:58:57,269 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into cca2c5b9952b40a4a7a70194daf043ee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:57,269 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:57,269 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=13, startTime=1731689936822; duration=0sec 2024-11-15T16:58:57,269 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:57,269 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:58:57,269 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 237a17164453419c8ee5c688f48ae7bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731689936314 2024-11-15T16:58:57,276 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:57,276 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/907ff38cd9f44c77908c34d501a0baf1 is 50, key is test_row_0/C:col10/1731689936314/Put/seqid=0 2024-11-15T16:58:57,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742067_1243 (size=12207) 2024-11-15T16:58:57,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689997325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689997326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689997326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689997336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689997336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T16:58:57,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T16:58:57,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:57,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:57,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:57,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/b85b2e433d8e499ebe0760df59f4c5d3 2024-11-15T16:58:57,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/43e4245998c54526a34c1c941f75d90c is 50, key is test_row_0/C:col10/1731689936365/Put/seqid=0 2024-11-15T16:58:57,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742068_1244 (size=12001) 2024-11-15T16:58:57,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/43e4245998c54526a34c1c941f75d90c 2024-11-15T16:58:57,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/6f7fa71f0e0442d68db586680cb5ebc6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6f7fa71f0e0442d68db586680cb5ebc6 2024-11-15T16:58:57,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6f7fa71f0e0442d68db586680cb5ebc6, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T16:58:57,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/b85b2e433d8e499ebe0760df59f4c5d3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/b85b2e433d8e499ebe0760df59f4c5d3 2024-11-15T16:58:57,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/b85b2e433d8e499ebe0760df59f4c5d3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T16:58:57,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/43e4245998c54526a34c1c941f75d90c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/43e4245998c54526a34c1c941f75d90c 2024-11-15T16:58:57,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/43e4245998c54526a34c1c941f75d90c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T16:58:57,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for e122d8e8e1d3a6c89c4dc261c769f99f in 459ms, sequenceid=115, compaction requested=false 2024-11-15T16:58:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:57,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T16:58:57,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:57,549 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:58:57,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/5878ae0657804be0927dae4e651c4925 is 50, key is test_row_0/A:col10/1731689937006/Put/seqid=0 2024-11-15T16:58:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T16:58:57,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742069_1245 (size=12001) 2024-11-15T16:58:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:57,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:57,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689997666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689997667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689997668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689997669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689997669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,687 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/907ff38cd9f44c77908c34d501a0baf1 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/907ff38cd9f44c77908c34d501a0baf1 2024-11-15T16:58:57,692 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into 907ff38cd9f44c77908c34d501a0baf1(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:57,692 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:57,692 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=13, startTime=1731689936822; duration=0sec 2024-11-15T16:58:57,692 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:57,692 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:58:57,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689997775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689997775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689997775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689997776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689997786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T16:58:57,960 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/5878ae0657804be0927dae4e651c4925 2024-11-15T16:58:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5ce564feb6844213b3a5152108e5b51b is 50, key is test_row_0/B:col10/1731689937006/Put/seqid=0 2024-11-15T16:58:57,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742070_1246 (size=12001) 2024-11-15T16:58:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689997986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689997986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689997986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689997986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689997995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689998295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689998296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689998296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689998296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689998306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T16:58:58,380 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5ce564feb6844213b3a5152108e5b51b 2024-11-15T16:58:58,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/6cdf4e31b19740fc8b47dc56fe01b8d6 is 50, key is test_row_0/C:col10/1731689937006/Put/seqid=0 2024-11-15T16:58:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742071_1247 (size=12001) 2024-11-15T16:58:58,793 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/6cdf4e31b19740fc8b47dc56fe01b8d6 2024-11-15T16:58:58,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/5878ae0657804be0927dae4e651c4925 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/5878ae0657804be0927dae4e651c4925 2024-11-15T16:58:58,802 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/5878ae0657804be0927dae4e651c4925, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T16:58:58,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5ce564feb6844213b3a5152108e5b51b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5ce564feb6844213b3a5152108e5b51b 2024-11-15T16:58:58,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689998806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689998806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,808 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5ce564feb6844213b3a5152108e5b51b, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T16:58:58,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689998807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689998807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/6cdf4e31b19740fc8b47dc56fe01b8d6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/6cdf4e31b19740fc8b47dc56fe01b8d6 2024-11-15T16:58:58,813 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/6cdf4e31b19740fc8b47dc56fe01b8d6, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T16:58:58,814 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e122d8e8e1d3a6c89c4dc261c769f99f in 1265ms, sequenceid=128, compaction requested=true 2024-11-15T16:58:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T16:58:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T16:58:58,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T16:58:58,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-15T16:58:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:58,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:58:58,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T16:58:58,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5910 sec 2024-11-15T16:58:58,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/8c5ab64f08d64115a7d940e0d91dd241 is 50, key is test_row_0/A:col10/1731689937667/Put/seqid=0 2024-11-15T16:58:58,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742072_1248 (size=12151) 2024-11-15T16:58:58,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689998848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:58,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689998966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689999176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/8c5ab64f08d64115a7d940e0d91dd241 2024-11-15T16:58:59,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/9ee0952a2825460c8124f43fab64ce09 is 50, key is test_row_0/B:col10/1731689937667/Put/seqid=0 2024-11-15T16:58:59,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742073_1249 (size=12151) 2024-11-15T16:58:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T16:58:59,386 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:58:59,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:58:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T16:58:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T16:58:59,388 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:58:59,389 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:58:59,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:58:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731689999485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T16:58:59,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T16:58:59,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:59,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:59,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:59,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:59,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:58:59,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/9ee0952a2825460c8124f43fab64ce09 2024-11-15T16:58:59,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/2ecbe1fbe44c4ed182582ae4c4b2f04a is 50, key is test_row_0/C:col10/1731689937667/Put/seqid=0 2024-11-15T16:58:59,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742074_1250 (size=12151) 2024-11-15T16:58:59,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/2ecbe1fbe44c4ed182582ae4c4b2f04a 2024-11-15T16:58:59,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/8c5ab64f08d64115a7d940e0d91dd241 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/8c5ab64f08d64115a7d940e0d91dd241 2024-11-15T16:58:59,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/8c5ab64f08d64115a7d940e0d91dd241, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T16:58:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/9ee0952a2825460c8124f43fab64ce09 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/9ee0952a2825460c8124f43fab64ce09 2024-11-15T16:58:59,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/9ee0952a2825460c8124f43fab64ce09, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T16:58:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/2ecbe1fbe44c4ed182582ae4c4b2f04a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2ecbe1fbe44c4ed182582ae4c4b2f04a 2024-11-15T16:58:59,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2ecbe1fbe44c4ed182582ae4c4b2f04a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T16:58:59,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e122d8e8e1d3a6c89c4dc261c769f99f in 863ms, sequenceid=155, compaction requested=true 2024-11-15T16:58:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:58:59,682 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:58:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:59,682 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:58:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:59,683 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:59,683 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:59,683 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:58:59,683 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:58:59,683 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:59,683 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:59,683 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cca2c5b9952b40a4a7a70194daf043ee, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/b85b2e433d8e499ebe0760df59f4c5d3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5ce564feb6844213b3a5152108e5b51b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/9ee0952a2825460c8124f43fab64ce09] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=47.2 K 2024-11-15T16:58:59,683 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/78cb1caf353b494e95e9e907a948e3bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6f7fa71f0e0442d68db586680cb5ebc6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/5878ae0657804be0927dae4e651c4925, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/8c5ab64f08d64115a7d940e0d91dd241] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=47.2 K 2024-11-15T16:58:59,685 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting cca2c5b9952b40a4a7a70194daf043ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731689936314 2024-11-15T16:58:59,685 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78cb1caf353b494e95e9e907a948e3bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731689936314 2024-11-15T16:58:59,685 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f7fa71f0e0442d68db586680cb5ebc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731689936349 2024-11-15T16:58:59,685 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting b85b2e433d8e499ebe0760df59f4c5d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731689936349 2024-11-15T16:58:59,685 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ce564feb6844213b3a5152108e5b51b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731689937003 2024-11-15T16:58:59,685 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5878ae0657804be0927dae4e651c4925, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731689937003 2024-11-15T16:58:59,686 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ee0952a2825460c8124f43fab64ce09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689937654 2024-11-15T16:58:59,686 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c5ab64f08d64115a7d940e0d91dd241, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689937654 2024-11-15T16:58:59,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T16:58:59,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:59,697 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T16:58:59,702 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#207 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:59,702 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/35962ad5577746f095380be78dd5ffe5 is 50, key is test_row_0/A:col10/1731689937667/Put/seqid=0 2024-11-15T16:58:59,702 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:59,703 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/e30e37ecf7fe42c298f007baa69f3d76 is 50, key is test_row_0/B:col10/1731689937667/Put/seqid=0 2024-11-15T16:58:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T16:58:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/e970b289fd3540599e7c517192e20f4a is 50, key is test_row_0/A:col10/1731689938846/Put/seqid=0 2024-11-15T16:58:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742077_1253 (size=12151) 2024-11-15T16:58:59,728 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/e970b289fd3540599e7c517192e20f4a 2024-11-15T16:58:59,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742076_1252 (size=12493) 2024-11-15T16:58:59,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742075_1251 (size=12493) 2024-11-15T16:58:59,740 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/35962ad5577746f095380be78dd5ffe5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/35962ad5577746f095380be78dd5ffe5 2024-11-15T16:58:59,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/1d13fd7c478c438080b1ff36ce4b96b0 is 50, key is test_row_0/B:col10/1731689938846/Put/seqid=0 2024-11-15T16:58:59,746 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/e30e37ecf7fe42c298f007baa69f3d76 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e30e37ecf7fe42c298f007baa69f3d76 2024-11-15T16:58:59,751 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 35962ad5577746f095380be78dd5ffe5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:59,751 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:59,751 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=12, startTime=1731689939682; duration=0sec 2024-11-15T16:58:59,751 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:58:59,752 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:58:59,752 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:58:59,755 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into e30e37ecf7fe42c298f007baa69f3d76(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:59,755 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:59,755 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=12, startTime=1731689939682; duration=0sec 2024-11-15T16:58:59,755 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:59,755 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:58:59,756 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:58:59,756 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:58:59,756 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:58:59,756 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/907ff38cd9f44c77908c34d501a0baf1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/43e4245998c54526a34c1c941f75d90c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/6cdf4e31b19740fc8b47dc56fe01b8d6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2ecbe1fbe44c4ed182582ae4c4b2f04a] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=47.2 K 2024-11-15T16:58:59,756 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907ff38cd9f44c77908c34d501a0baf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731689936314 2024-11-15T16:58:59,757 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e4245998c54526a34c1c941f75d90c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731689936349 2024-11-15T16:58:59,757 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cdf4e31b19740fc8b47dc56fe01b8d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731689937003 2024-11-15T16:58:59,758 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ecbe1fbe44c4ed182582ae4c4b2f04a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689937654 2024-11-15T16:58:59,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742078_1254 (size=12151) 2024-11-15T16:58:59,771 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/1d13fd7c478c438080b1ff36ce4b96b0 2024-11-15T16:58:59,782 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:58:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d64fab27e73b44c8a8b5f7f548bd6e63 is 50, key is test_row_0/C:col10/1731689938846/Put/seqid=0 2024-11-15T16:58:59,782 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/20c72ab3daf043e0bf19ea01f0b0bf03 is 50, key is test_row_0/C:col10/1731689937667/Put/seqid=0 2024-11-15T16:58:59,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742079_1255 (size=12151) 2024-11-15T16:58:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:58:59,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:58:59,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742080_1256 (size=12493) 2024-11-15T16:58:59,828 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/20c72ab3daf043e0bf19ea01f0b0bf03 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/20c72ab3daf043e0bf19ea01f0b0bf03 2024-11-15T16:58:59,834 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into 20c72ab3daf043e0bf19ea01f0b0bf03(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:58:59,834 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:58:59,834 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=12, startTime=1731689939682; duration=0sec 2024-11-15T16:58:59,834 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:58:59,834 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:58:59,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689999855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689999859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689999861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689999863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731689999966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731689999966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731689999975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:58:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:58:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731689999975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:58:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:58:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731690000004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T16:59:00,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690000176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690000176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690000185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690000185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,203 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d64fab27e73b44c8a8b5f7f548bd6e63 2024-11-15T16:59:00,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/e970b289fd3540599e7c517192e20f4a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e970b289fd3540599e7c517192e20f4a 2024-11-15T16:59:00,212 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e970b289fd3540599e7c517192e20f4a, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T16:59:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/1d13fd7c478c438080b1ff36ce4b96b0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d13fd7c478c438080b1ff36ce4b96b0 2024-11-15T16:59:00,216 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d13fd7c478c438080b1ff36ce4b96b0, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T16:59:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d64fab27e73b44c8a8b5f7f548bd6e63 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d64fab27e73b44c8a8b5f7f548bd6e63 2024-11-15T16:59:00,221 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d64fab27e73b44c8a8b5f7f548bd6e63, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T16:59:00,222 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e122d8e8e1d3a6c89c4dc261c769f99f in 526ms, sequenceid=165, compaction requested=false 2024-11-15T16:59:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T16:59:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T16:59:00,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T16:59:00,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 835 msec 2024-11-15T16:59:00,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 840 msec 2024-11-15T16:59:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:00,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:00,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T16:59:00,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/340c64aea5954c418d5cc02b137a561b is 50, key is test_row_0/A:col10/1731689939861/Put/seqid=0 2024-11-15T16:59:00,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690000496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742081_1257 (size=14541) 2024-11-15T16:59:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690000497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690000500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690000501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T16:59:00,526 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:00,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T16:59:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T16:59:00,529 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:00,529 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:00,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:00,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690000605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690000615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690000616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T16:59:00,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T16:59:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:00,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690000815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690000825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690000826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:00,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T16:59:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:00,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T16:59:00,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/340c64aea5954c418d5cc02b137a561b 2024-11-15T16:59:00,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6f596990ee914c18b23d7ca1d988be04 is 50, key is test_row_0/B:col10/1731689939861/Put/seqid=0 2024-11-15T16:59:00,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742082_1258 (size=12151) 2024-11-15T16:59:00,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T16:59:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:00,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:01,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690001005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731690001015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,017 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690001125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690001136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690001136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T16:59:01,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:01,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:01,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T16:59:01,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T16:59:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:01,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6f596990ee914c18b23d7ca1d988be04 2024-11-15T16:59:01,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/03afb06361ad4228a58def5fd77f4f95 is 50, key is test_row_0/C:col10/1731689939861/Put/seqid=0 2024-11-15T16:59:01,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742083_1259 (size=12151) 2024-11-15T16:59:01,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/03afb06361ad4228a58def5fd77f4f95 2024-11-15T16:59:01,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/340c64aea5954c418d5cc02b137a561b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/340c64aea5954c418d5cc02b137a561b 2024-11-15T16:59:01,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/340c64aea5954c418d5cc02b137a561b, entries=200, sequenceid=196, filesize=14.2 K 2024-11-15T16:59:01,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/6f596990ee914c18b23d7ca1d988be04 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6f596990ee914c18b23d7ca1d988be04 2024-11-15T16:59:01,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6f596990ee914c18b23d7ca1d988be04, entries=150, sequenceid=196, filesize=11.9 K 2024-11-15T16:59:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/03afb06361ad4228a58def5fd77f4f95 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/03afb06361ad4228a58def5fd77f4f95 2024-11-15T16:59:01,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/03afb06361ad4228a58def5fd77f4f95, entries=150, sequenceid=196, filesize=11.9 K 2024-11-15T16:59:01,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for e122d8e8e1d3a6c89c4dc261c769f99f in 886ms, sequenceid=196, compaction requested=true 2024-11-15T16:59:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:01,374 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:01,375 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:01,376 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:01,376 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:59:01,376 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:01,376 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,376 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:59:01,376 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,376 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/35962ad5577746f095380be78dd5ffe5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e970b289fd3540599e7c517192e20f4a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/340c64aea5954c418d5cc02b137a561b] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=38.3 K 2024-11-15T16:59:01,376 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e30e37ecf7fe42c298f007baa69f3d76, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d13fd7c478c438080b1ff36ce4b96b0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6f596990ee914c18b23d7ca1d988be04] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=35.9 K 2024-11-15T16:59:01,377 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35962ad5577746f095380be78dd5ffe5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689937654 2024-11-15T16:59:01,377 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e30e37ecf7fe42c298f007baa69f3d76, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689937654 2024-11-15T16:59:01,377 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d13fd7c478c438080b1ff36ce4b96b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731689938838 2024-11-15T16:59:01,377 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e970b289fd3540599e7c517192e20f4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731689938838 2024-11-15T16:59:01,377 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f596990ee914c18b23d7ca1d988be04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731689939861 2024-11-15T16:59:01,377 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 340c64aea5954c418d5cc02b137a561b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731689939859 2024-11-15T16:59:01,386 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:01,386 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/38b140857d18485ba3d2cecb8fca5418 is 50, key is test_row_0/A:col10/1731689939861/Put/seqid=0 2024-11-15T16:59:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742084_1260 (size=12595) 2024-11-15T16:59:01,398 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:01,399 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/63a1f207180a4fec9ca5956cd3f8b2a4 is 50, key is test_row_0/B:col10/1731689939861/Put/seqid=0 2024-11-15T16:59:01,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742085_1261 (size=12595) 2024-11-15T16:59:01,406 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/38b140857d18485ba3d2cecb8fca5418 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/38b140857d18485ba3d2cecb8fca5418 2024-11-15T16:59:01,411 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 38b140857d18485ba3d2cecb8fca5418(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:01,411 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:01,411 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=13, startTime=1731689941374; duration=0sec 2024-11-15T16:59:01,411 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:01,411 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:59:01,412 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:01,413 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:01,413 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:59:01,413 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,413 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/20c72ab3daf043e0bf19ea01f0b0bf03, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d64fab27e73b44c8a8b5f7f548bd6e63, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/03afb06361ad4228a58def5fd77f4f95] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=35.9 K 2024-11-15T16:59:01,414 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c72ab3daf043e0bf19ea01f0b0bf03, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731689937654 2024-11-15T16:59:01,414 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64fab27e73b44c8a8b5f7f548bd6e63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731689938838 2024-11-15T16:59:01,415 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03afb06361ad4228a58def5fd77f4f95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731689939861 2024-11-15T16:59:01,423 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:01,423 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/9671758b64de41fda5a47057be01843d is 50, key is test_row_0/C:col10/1731689939861/Put/seqid=0 2024-11-15T16:59:01,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742086_1262 (size=12595) 2024-11-15T16:59:01,440 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/9671758b64de41fda5a47057be01843d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9671758b64de41fda5a47057be01843d 2024-11-15T16:59:01,447 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into 9671758b64de41fda5a47057be01843d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:01,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:01,447 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=13, startTime=1731689941374; duration=0sec 2024-11-15T16:59:01,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:01,447 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:59:01,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T16:59:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:01,449 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T16:59:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/2268ecf0dd70433681edef8300e109e6 is 50, key is test_row_0/A:col10/1731689940498/Put/seqid=0 2024-11-15T16:59:01,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742087_1263 (size=12151) 2024-11-15T16:59:01,474 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/2268ecf0dd70433681edef8300e109e6 2024-11-15T16:59:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/988444757b364ca5843adf4e6a608733 is 50, key is test_row_0/B:col10/1731689940498/Put/seqid=0 2024-11-15T16:59:01,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742088_1264 (size=12151) 2024-11-15T16:59:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T16:59:01,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690001684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690001686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690001686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690001795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690001795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690001795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:01,810 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/63a1f207180a4fec9ca5956cd3f8b2a4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/63a1f207180a4fec9ca5956cd3f8b2a4 2024-11-15T16:59:01,815 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into 63a1f207180a4fec9ca5956cd3f8b2a4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:01,815 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:01,815 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=13, startTime=1731689941374; duration=0sec 2024-11-15T16:59:01,815 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:01,815 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:59:01,816 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T16:59:01,816 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T16:59:01,889 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/988444757b364ca5843adf4e6a608733 2024-11-15T16:59:01,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/f44749c167e4406889a595b438e97972 is 50, key is test_row_0/C:col10/1731689940498/Put/seqid=0 2024-11-15T16:59:01,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742089_1265 (size=12151) 2024-11-15T16:59:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690002006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690002006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690002006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690002025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,027 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_0' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,301 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/f44749c167e4406889a595b438e97972 2024-11-15T16:59:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/2268ecf0dd70433681edef8300e109e6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/2268ecf0dd70433681edef8300e109e6 2024-11-15T16:59:02,311 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/2268ecf0dd70433681edef8300e109e6, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T16:59:02,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/988444757b364ca5843adf4e6a608733 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/988444757b364ca5843adf4e6a608733 2024-11-15T16:59:02,317 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/988444757b364ca5843adf4e6a608733, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T16:59:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/f44749c167e4406889a595b438e97972 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f44749c167e4406889a595b438e97972 2024-11-15T16:59:02,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690002316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690002316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690002316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,323 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f44749c167e4406889a595b438e97972, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T16:59:02,324 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for e122d8e8e1d3a6c89c4dc261c769f99f in 874ms, sequenceid=206, compaction requested=false 2024-11-15T16:59:02,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:02,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:02,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T16:59:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T16:59:02,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T16:59:02,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7960 sec 2024-11-15T16:59:02,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8010 sec 2024-11-15T16:59:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T16:59:02,676 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:02,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T16:59:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T16:59:02,678 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:02,679 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:02,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T16:59:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:02,829 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:02,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-15T16:59:02,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T16:59:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:02,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:02,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690002831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/7283825b2db54243b9aa6ab772c3e69c is 50, key is test_row_0/A:col10/1731689942829/Put/seqid=0 2024-11-15T16:59:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690002834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690002836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742090_1266 (size=14541) 2024-11-15T16:59:02,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690002946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690002946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690002946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:02,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T16:59:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:02,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T16:59:03,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731690003036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:03,038 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:03,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T16:59:03,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:03,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690003156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:03,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690003156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690003156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:03,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/7283825b2db54243b9aa6ab772c3e69c 2024-11-15T16:59:03,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/36451dbcfa6a498b9b146d84b0648de0 is 50, key is test_row_0/B:col10/1731689942829/Put/seqid=0 2024-11-15T16:59:03,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742091_1267 (size=12151) 2024-11-15T16:59:03,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T16:59:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:03,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T16:59:03,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T16:59:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:03,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690003466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:03,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690003466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:03,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690003466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:03,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T16:59:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:03,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/36451dbcfa6a498b9b146d84b0648de0 2024-11-15T16:59:03,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/446785c65d2a44ad9ab311cfd1cac930 is 50, key is test_row_0/C:col10/1731689942829/Put/seqid=0 2024-11-15T16:59:03,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742092_1268 (size=12151) 2024-11-15T16:59:03,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/446785c65d2a44ad9ab311cfd1cac930 2024-11-15T16:59:03,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/7283825b2db54243b9aa6ab772c3e69c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/7283825b2db54243b9aa6ab772c3e69c 2024-11-15T16:59:03,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/7283825b2db54243b9aa6ab772c3e69c, entries=200, sequenceid=238, filesize=14.2 K 2024-11-15T16:59:03,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/36451dbcfa6a498b9b146d84b0648de0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/36451dbcfa6a498b9b146d84b0648de0 2024-11-15T16:59:03,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/36451dbcfa6a498b9b146d84b0648de0, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T16:59:03,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/446785c65d2a44ad9ab311cfd1cac930 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/446785c65d2a44ad9ab311cfd1cac930 2024-11-15T16:59:03,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/446785c65d2a44ad9ab311cfd1cac930, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T16:59:03,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for e122d8e8e1d3a6c89c4dc261c769f99f in 858ms, sequenceid=238, compaction requested=true 2024-11-15T16:59:03,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:03,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:03,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:03,687 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:03,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:03,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:03,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:03,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:03,687 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:03,688 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:03,688 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:03,689 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:59:03,689 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:59:03,689 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,689 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,689 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/38b140857d18485ba3d2cecb8fca5418, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/2268ecf0dd70433681edef8300e109e6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/7283825b2db54243b9aa6ab772c3e69c] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=38.4 K 2024-11-15T16:59:03,689 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/63a1f207180a4fec9ca5956cd3f8b2a4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/988444757b364ca5843adf4e6a608733, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/36451dbcfa6a498b9b146d84b0648de0] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=36.0 K 2024-11-15T16:59:03,690 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b140857d18485ba3d2cecb8fca5418, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731689939861 2024-11-15T16:59:03,690 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a1f207180a4fec9ca5956cd3f8b2a4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731689939861 2024-11-15T16:59:03,690 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2268ecf0dd70433681edef8300e109e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689940493 2024-11-15T16:59:03,690 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 988444757b364ca5843adf4e6a608733, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689940493 2024-11-15T16:59:03,691 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7283825b2db54243b9aa6ab772c3e69c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731689941684 2024-11-15T16:59:03,691 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 36451dbcfa6a498b9b146d84b0648de0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731689941684 2024-11-15T16:59:03,700 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:03,700 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:03,700 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/1e662a87fa944aa5aca89cfc1880a13a is 50, key is test_row_0/A:col10/1731689942829/Put/seqid=0 2024-11-15T16:59:03,700 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/cfe015a1d1ec403ab523587da2ba6c33 is 50, key is test_row_0/B:col10/1731689942829/Put/seqid=0 2024-11-15T16:59:03,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742094_1270 (size=12697) 2024-11-15T16:59:03,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742093_1269 (size=12697) 2024-11-15T16:59:03,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T16:59:03,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:03,749 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-15T16:59:03,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/9bf6ecbaaaf34793a11bbdd213dc3410 is 50, key is test_row_0/A:col10/1731689942833/Put/seqid=0 2024-11-15T16:59:03,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742095_1271 (size=7365) 2024-11-15T16:59:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T16:59:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:04,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690004020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690004022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690004023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690004035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,038 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_0' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,125 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/cfe015a1d1ec403ab523587da2ba6c33 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cfe015a1d1ec403ab523587da2ba6c33 2024-11-15T16:59:04,127 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/1e662a87fa944aa5aca89cfc1880a13a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e662a87fa944aa5aca89cfc1880a13a 2024-11-15T16:59:04,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690004126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,132 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into cfe015a1d1ec403ab523587da2ba6c33(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:04,132 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:04,132 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 1e662a87fa944aa5aca89cfc1880a13a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:04,132 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=13, startTime=1731689943687; duration=0sec 2024-11-15T16:59:04,132 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:04,132 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=13, startTime=1731689943687; duration=0sec 2024-11-15T16:59:04,132 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:04,132 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:59:04,132 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:04,132 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:59:04,132 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:04,133 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:04,133 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:59:04,133 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:04,133 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9671758b64de41fda5a47057be01843d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f44749c167e4406889a595b438e97972, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/446785c65d2a44ad9ab311cfd1cac930] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=36.0 K 2024-11-15T16:59:04,134 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 9671758b64de41fda5a47057be01843d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731689939861 2024-11-15T16:59:04,134 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f44749c167e4406889a595b438e97972, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731689940493 2024-11-15T16:59:04,134 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 446785c65d2a44ad9ab311cfd1cac930, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731689941684 2024-11-15T16:59:04,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690004136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690004140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,142 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:04,142 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d6edb73dcc9f4a9b898e2793b50077c7 is 50, key is test_row_0/C:col10/1731689942829/Put/seqid=0 2024-11-15T16:59:04,160 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/9bf6ecbaaaf34793a11bbdd213dc3410 2024-11-15T16:59:04,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/525acfdc8bb340669078b9fa5c4a1f00 is 50, key is test_row_0/B:col10/1731689942833/Put/seqid=0 2024-11-15T16:59:04,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742097_1273 (size=7365) 2024-11-15T16:59:04,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742096_1272 (size=12697) 2024-11-15T16:59:04,179 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/525acfdc8bb340669078b9fa5c4a1f00 2024-11-15T16:59:04,190 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d6edb73dcc9f4a9b898e2793b50077c7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d6edb73dcc9f4a9b898e2793b50077c7 2024-11-15T16:59:04,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/227a4a0c9bc042d881a2384941fb4b7a is 50, key is test_row_0/C:col10/1731689942833/Put/seqid=0 2024-11-15T16:59:04,199 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into d6edb73dcc9f4a9b898e2793b50077c7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:04,199 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:04,199 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=13, startTime=1731689943687; duration=0sec 2024-11-15T16:59:04,199 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:04,199 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:59:04,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742098_1274 (size=7365) 2024-11-15T16:59:04,202 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/227a4a0c9bc042d881a2384941fb4b7a 2024-11-15T16:59:04,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/9bf6ecbaaaf34793a11bbdd213dc3410 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9bf6ecbaaaf34793a11bbdd213dc3410 2024-11-15T16:59:04,211 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9bf6ecbaaaf34793a11bbdd213dc3410, entries=50, sequenceid=243, filesize=7.2 K 2024-11-15T16:59:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/525acfdc8bb340669078b9fa5c4a1f00 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/525acfdc8bb340669078b9fa5c4a1f00 2024-11-15T16:59:04,217 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/525acfdc8bb340669078b9fa5c4a1f00, entries=50, sequenceid=243, filesize=7.2 K 2024-11-15T16:59:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/227a4a0c9bc042d881a2384941fb4b7a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/227a4a0c9bc042d881a2384941fb4b7a 2024-11-15T16:59:04,224 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/227a4a0c9bc042d881a2384941fb4b7a, entries=50, sequenceid=243, filesize=7.2 K 2024-11-15T16:59:04,225 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for e122d8e8e1d3a6c89c4dc261c769f99f in 476ms, sequenceid=243, compaction requested=false 2024-11-15T16:59:04,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:04,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:04,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T16:59:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T16:59:04,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T16:59:04,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5470 sec 2024-11-15T16:59:04,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5530 sec 2024-11-15T16:59:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:04,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:04,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-15T16:59:04,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/cf2f9988aa1c4567a15b285173d44684 is 50, key is test_row_0/A:col10/1731689944021/Put/seqid=0 2024-11-15T16:59:04,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690004342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690004346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690004346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742099_1275 (size=14741) 2024-11-15T16:59:04,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/cf2f9988aa1c4567a15b285173d44684 2024-11-15T16:59:04,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/04449fcdf3fb4bff8a69da19705e6dd4 is 50, key is test_row_0/B:col10/1731689944021/Put/seqid=0 2024-11-15T16:59:04,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742100_1276 (size=12301) 2024-11-15T16:59:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690004456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690004656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690004656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690004665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/04449fcdf3fb4bff8a69da19705e6dd4 2024-11-15T16:59:04,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/e72032b4255f4837881bd3681ddd299d is 50, key is test_row_0/C:col10/1731689944021/Put/seqid=0 2024-11-15T16:59:04,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742101_1277 (size=12301) 2024-11-15T16:59:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T16:59:04,826 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:04,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-15T16:59:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T16:59:04,828 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:04,829 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:04,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:04,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T16:59:04,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690004976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:04,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T16:59:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:04,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:05,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T16:59:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:05,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T16:59:05,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690005165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690005165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:05,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/e72032b4255f4837881bd3681ddd299d 2024-11-15T16:59:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/cf2f9988aa1c4567a15b285173d44684 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cf2f9988aa1c4567a15b285173d44684 2024-11-15T16:59:05,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cf2f9988aa1c4567a15b285173d44684, entries=200, sequenceid=278, filesize=14.4 K 2024-11-15T16:59:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/04449fcdf3fb4bff8a69da19705e6dd4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/04449fcdf3fb4bff8a69da19705e6dd4 2024-11-15T16:59:05,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/04449fcdf3fb4bff8a69da19705e6dd4, entries=150, sequenceid=278, filesize=12.0 K 2024-11-15T16:59:05,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/e72032b4255f4837881bd3681ddd299d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e72032b4255f4837881bd3681ddd299d 2024-11-15T16:59:05,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e72032b4255f4837881bd3681ddd299d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-15T16:59:05,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for e122d8e8e1d3a6c89c4dc261c769f99f in 865ms, sequenceid=278, compaction requested=true 2024-11-15T16:59:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:05,204 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:05,204 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:05,205 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34803 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:05,205 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:59:05,205 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:05,205 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e662a87fa944aa5aca89cfc1880a13a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9bf6ecbaaaf34793a11bbdd213dc3410, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cf2f9988aa1c4567a15b285173d44684] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=34.0 K 2024-11-15T16:59:05,205 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:05,205 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:59:05,205 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:05,206 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cfe015a1d1ec403ab523587da2ba6c33, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/525acfdc8bb340669078b9fa5c4a1f00, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/04449fcdf3fb4bff8a69da19705e6dd4] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=31.6 K 2024-11-15T16:59:05,206 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e662a87fa944aa5aca89cfc1880a13a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731689941684 2024-11-15T16:59:05,206 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting cfe015a1d1ec403ab523587da2ba6c33, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731689941684 2024-11-15T16:59:05,206 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bf6ecbaaaf34793a11bbdd213dc3410, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731689942833 2024-11-15T16:59:05,206 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 525acfdc8bb340669078b9fa5c4a1f00, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731689942833 2024-11-15T16:59:05,207 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2f9988aa1c4567a15b285173d44684, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731689944016 2024-11-15T16:59:05,207 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 04449fcdf3fb4bff8a69da19705e6dd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731689944021 2024-11-15T16:59:05,215 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:05,215 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:05,216 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/14c29f83acd94ca9a52968e8a1dade5f is 50, key is test_row_0/A:col10/1731689944021/Put/seqid=0 2024-11-15T16:59:05,216 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/332d210532bf4e71a379595ddf5a8148 is 50, key is test_row_0/B:col10/1731689944021/Put/seqid=0 2024-11-15T16:59:05,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742102_1278 (size=12949) 2024-11-15T16:59:05,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742103_1279 (size=12949) 2024-11-15T16:59:05,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T16:59:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:05,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-15T16:59:05,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/34303a21cc6b45a7b0a85e24624c4343 is 50, key is test_row_1/A:col10/1731689944340/Put/seqid=0 2024-11-15T16:59:05,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742104_1280 (size=7415) 2024-11-15T16:59:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T16:59:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:05,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:05,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690005581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:05,628 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/14c29f83acd94ca9a52968e8a1dade5f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/14c29f83acd94ca9a52968e8a1dade5f 2024-11-15T16:59:05,634 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/332d210532bf4e71a379595ddf5a8148 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/332d210532bf4e71a379595ddf5a8148 2024-11-15T16:59:05,635 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 14c29f83acd94ca9a52968e8a1dade5f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:05,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:05,635 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=13, startTime=1731689945203; duration=0sec 2024-11-15T16:59:05,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:05,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:59:05,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:05,636 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:05,636 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:59:05,636 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:05,636 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d6edb73dcc9f4a9b898e2793b50077c7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/227a4a0c9bc042d881a2384941fb4b7a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e72032b4255f4837881bd3681ddd299d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=31.6 K 2024-11-15T16:59:05,637 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6edb73dcc9f4a9b898e2793b50077c7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731689941684 2024-11-15T16:59:05,637 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 227a4a0c9bc042d881a2384941fb4b7a, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731689942833 2024-11-15T16:59:05,638 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e72032b4255f4837881bd3681ddd299d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731689944021 2024-11-15T16:59:05,639 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into 332d210532bf4e71a379595ddf5a8148(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:05,639 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:05,639 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=13, startTime=1731689945204; duration=0sec 2024-11-15T16:59:05,639 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:05,639 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:59:05,646 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:05,646 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/18b65f426323424483d9afd657bff24a is 50, key is test_row_0/C:col10/1731689944021/Put/seqid=0 2024-11-15T16:59:05,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742105_1281 (size=12949) 2024-11-15T16:59:05,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690005686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:05,699 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/34303a21cc6b45a7b0a85e24624c4343 2024-11-15T16:59:05,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/1d788cc649a44fe5baed929e21042e56 is 50, key is test_row_1/B:col10/1731689944340/Put/seqid=0 2024-11-15T16:59:05,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742106_1282 (size=7415) 2024-11-15T16:59:05,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690005895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T16:59:06,060 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/18b65f426323424483d9afd657bff24a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/18b65f426323424483d9afd657bff24a 2024-11-15T16:59:06,066 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into 18b65f426323424483d9afd657bff24a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:06,066 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:06,066 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=13, startTime=1731689945204; duration=0sec 2024-11-15T16:59:06,066 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:06,066 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:59:06,111 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/1d788cc649a44fe5baed929e21042e56 2024-11-15T16:59:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/065137dd3733472ca48241d2ae293ea4 is 50, key is test_row_1/C:col10/1731689944340/Put/seqid=0 2024-11-15T16:59:06,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742107_1283 (size=7415) 2024-11-15T16:59:06,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690006176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:06,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690006176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:06,177 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:06,178 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:06,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690006205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:06,527 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/065137dd3733472ca48241d2ae293ea4 2024-11-15T16:59:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/34303a21cc6b45a7b0a85e24624c4343 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/34303a21cc6b45a7b0a85e24624c4343 2024-11-15T16:59:06,535 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/34303a21cc6b45a7b0a85e24624c4343, entries=50, sequenceid=282, filesize=7.2 K 2024-11-15T16:59:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/1d788cc649a44fe5baed929e21042e56 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d788cc649a44fe5baed929e21042e56 2024-11-15T16:59:06,540 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d788cc649a44fe5baed929e21042e56, entries=50, sequenceid=282, filesize=7.2 K 2024-11-15T16:59:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/065137dd3733472ca48241d2ae293ea4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/065137dd3733472ca48241d2ae293ea4 2024-11-15T16:59:06,545 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/065137dd3733472ca48241d2ae293ea4, entries=50, sequenceid=282, filesize=7.2 K 2024-11-15T16:59:06,546 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=194.56 KB/199230 for e122d8e8e1d3a6c89c4dc261c769f99f in 1258ms, sequenceid=282, compaction requested=false 2024-11-15T16:59:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-15T16:59:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-15T16:59:06,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T16:59:06,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-15T16:59:06,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7240 sec 2024-11-15T16:59:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:06,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-15T16:59:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690006722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:06,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/792ad495b4924786869d17baf67ddc81 is 50, key is test_row_0/A:col10/1731689945580/Put/seqid=0 2024-11-15T16:59:06,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742108_1284 (size=14741) 2024-11-15T16:59:06,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690006835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T16:59:06,976 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:06,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-15T16:59:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T16:59:06,978 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:06,979 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:06,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690007046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1731690007075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:07,077 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T16:59:07,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T16:59:07,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/792ad495b4924786869d17baf67ddc81 2024-11-15T16:59:07,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:07,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/8086dd4bf52649569fc43c1fe2fd65c4 is 50, key is test_row_0/B:col10/1731689945580/Put/seqid=0 2024-11-15T16:59:07,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742109_1285 (size=12301) 2024-11-15T16:59:07,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T16:59:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T16:59:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690007356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:07,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T16:59:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/8086dd4bf52649569fc43c1fe2fd65c4 2024-11-15T16:59:07,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/110573bf16e24c4985830a078c32cfad is 50, key is test_row_0/C:col10/1731689945580/Put/seqid=0 2024-11-15T16:59:07,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742110_1286 (size=12301) 2024-11-15T16:59:07,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T16:59:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T16:59:07,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T16:59:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690007866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:07,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T16:59:07,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:07,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:07,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/110573bf16e24c4985830a078c32cfad 2024-11-15T16:59:07,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/792ad495b4924786869d17baf67ddc81 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/792ad495b4924786869d17baf67ddc81 2024-11-15T16:59:07,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/792ad495b4924786869d17baf67ddc81, entries=200, sequenceid=318, filesize=14.4 K 2024-11-15T16:59:07,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/8086dd4bf52649569fc43c1fe2fd65c4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/8086dd4bf52649569fc43c1fe2fd65c4 2024-11-15T16:59:07,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/8086dd4bf52649569fc43c1fe2fd65c4, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T16:59:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/110573bf16e24c4985830a078c32cfad as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/110573bf16e24c4985830a078c32cfad 2024-11-15T16:59:07,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/110573bf16e24c4985830a078c32cfad, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T16:59:07,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for e122d8e8e1d3a6c89c4dc261c769f99f in 1261ms, sequenceid=318, compaction requested=true 2024-11-15T16:59:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:07,982 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:07,982 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:07,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:07,983 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35105 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:07,983 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:07,983 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:59:07,983 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:59:07,983 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,983 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/14c29f83acd94ca9a52968e8a1dade5f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/34303a21cc6b45a7b0a85e24624c4343, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/792ad495b4924786869d17baf67ddc81] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=34.3 K 2024-11-15T16:59:07,983 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:07,984 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/332d210532bf4e71a379595ddf5a8148, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d788cc649a44fe5baed929e21042e56, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/8086dd4bf52649569fc43c1fe2fd65c4] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=31.9 K 2024-11-15T16:59:07,984 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c29f83acd94ca9a52968e8a1dade5f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731689944021 2024-11-15T16:59:07,984 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 332d210532bf4e71a379595ddf5a8148, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731689944021 2024-11-15T16:59:07,984 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34303a21cc6b45a7b0a85e24624c4343, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731689944340 2024-11-15T16:59:07,984 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d788cc649a44fe5baed929e21042e56, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731689944340 2024-11-15T16:59:07,985 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8086dd4bf52649569fc43c1fe2fd65c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731689945564 2024-11-15T16:59:07,985 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792ad495b4924786869d17baf67ddc81, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731689945564 2024-11-15T16:59:07,993 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:07,993 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/1e132ae8e2804965a2b69c004dc58d0f is 50, key is test_row_0/A:col10/1731689945580/Put/seqid=0 2024-11-15T16:59:07,998 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:07,998 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5bdd7b696f9e4ae29c14614b0aad907c is 50, key is test_row_0/B:col10/1731689945580/Put/seqid=0 2024-11-15T16:59:08,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742112_1288 (size=13051) 2024-11-15T16:59:08,016 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5bdd7b696f9e4ae29c14614b0aad907c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5bdd7b696f9e4ae29c14614b0aad907c 2024-11-15T16:59:08,021 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into 5bdd7b696f9e4ae29c14614b0aad907c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:08,021 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:08,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742111_1287 (size=13051) 2024-11-15T16:59:08,021 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=13, startTime=1731689947982; duration=0sec 2024-11-15T16:59:08,021 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:08,021 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:59:08,021 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:08,023 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:08,023 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:59:08,023 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,023 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/18b65f426323424483d9afd657bff24a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/065137dd3733472ca48241d2ae293ea4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/110573bf16e24c4985830a078c32cfad] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=31.9 K 2024-11-15T16:59:08,024 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b65f426323424483d9afd657bff24a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731689944021 2024-11-15T16:59:08,025 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 065137dd3733472ca48241d2ae293ea4, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731689944340 2024-11-15T16:59:08,025 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 110573bf16e24c4985830a078c32cfad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731689945564 2024-11-15T16:59:08,027 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/1e132ae8e2804965a2b69c004dc58d0f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e132ae8e2804965a2b69c004dc58d0f 2024-11-15T16:59:08,033 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 1e132ae8e2804965a2b69c004dc58d0f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:08,033 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:08,033 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=13, startTime=1731689947982; duration=0sec 2024-11-15T16:59:08,034 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:08,034 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:59:08,035 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:08,035 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/f4e46e648a4a46ba9a7dd461fe54c522 is 50, key is test_row_0/C:col10/1731689945580/Put/seqid=0 2024-11-15T16:59:08,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742113_1289 (size=13051) 2024-11-15T16:59:08,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T16:59:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-15T16:59:08,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-15T16:59:08,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T16:59:08,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0740 sec 2024-11-15T16:59:08,057 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/f4e46e648a4a46ba9a7dd461fe54c522 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f4e46e648a4a46ba9a7dd461fe54c522 2024-11-15T16:59:08,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0810 sec 2024-11-15T16:59:08,063 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into f4e46e648a4a46ba9a7dd461fe54c522(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:08,063 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:08,063 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=13, startTime=1731689947983; duration=0sec 2024-11-15T16:59:08,063 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:08,063 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:59:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:08,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:59:08,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/c72dead56252490b84e422655205d9a4 is 50, key is test_row_0/A:col10/1731689948074/Put/seqid=0 2024-11-15T16:59:08,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742114_1290 (size=12301) 2024-11-15T16:59:08,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/c72dead56252490b84e422655205d9a4 2024-11-15T16:59:08,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/df14dfa0767f46e4a6400e3a7f02107d is 50, key is test_row_0/B:col10/1731689948074/Put/seqid=0 2024-11-15T16:59:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T16:59:08,117 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:08,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-15T16:59:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T16:59:08,119 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:08,120 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:08,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:08,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742115_1291 (size=12301) 2024-11-15T16:59:08,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690008181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:08,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690008186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:08,188 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690008206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:08,209 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T16:59:08,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T16:59:08,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:08,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690008296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:08,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T16:59:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T16:59:08,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690008506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:08,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/df14dfa0767f46e4a6400e3a7f02107d 2024-11-15T16:59:08,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/2a3d4214cc4b4b8f84a5628d2c762d27 is 50, key is test_row_0/C:col10/1731689948074/Put/seqid=0 2024-11-15T16:59:08,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742116_1292 (size=12301) 2024-11-15T16:59:08,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T16:59:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T16:59:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T16:59:08,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690008815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:08,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690008876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:08,879 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:08,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T16:59:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:08,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:08,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/2a3d4214cc4b4b8f84a5628d2c762d27 2024-11-15T16:59:08,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/c72dead56252490b84e422655205d9a4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/c72dead56252490b84e422655205d9a4 2024-11-15T16:59:08,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/c72dead56252490b84e422655205d9a4, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:59:08,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/df14dfa0767f46e4a6400e3a7f02107d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/df14dfa0767f46e4a6400e3a7f02107d 2024-11-15T16:59:08,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/df14dfa0767f46e4a6400e3a7f02107d, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:59:08,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/2a3d4214cc4b4b8f84a5628d2c762d27 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2a3d4214cc4b4b8f84a5628d2c762d27 2024-11-15T16:59:08,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2a3d4214cc4b4b8f84a5628d2c762d27, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T16:59:08,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e122d8e8e1d3a6c89c4dc261c769f99f in 890ms, sequenceid=333, compaction requested=false 2024-11-15T16:59:08,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:09,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T16:59:09,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:09,039 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:59:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/e6133b05a90441bd8bf4e803e39fb9ef is 50, key is test_row_0/A:col10/1731689948148/Put/seqid=0 2024-11-15T16:59:09,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742117_1293 (size=12301) 2024-11-15T16:59:09,054 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/e6133b05a90441bd8bf4e803e39fb9ef 2024-11-15T16:59:09,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/436ae2d92cf643019c4e65abdcddcebd is 50, key is test_row_0/B:col10/1731689948148/Put/seqid=0 2024-11-15T16:59:09,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742118_1294 (size=12301) 2024-11-15T16:59:09,066 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/436ae2d92cf643019c4e65abdcddcebd 2024-11-15T16:59:09,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/b2b5ff78502a4347887090df62ba0699 is 50, key is test_row_0/C:col10/1731689948148/Put/seqid=0 2024-11-15T16:59:09,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742119_1295 (size=12301) 2024-11-15T16:59:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T16:59:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690009359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690009466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:09,480 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/b2b5ff78502a4347887090df62ba0699 2024-11-15T16:59:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/e6133b05a90441bd8bf4e803e39fb9ef as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e6133b05a90441bd8bf4e803e39fb9ef 2024-11-15T16:59:09,489 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e6133b05a90441bd8bf4e803e39fb9ef, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T16:59:09,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/436ae2d92cf643019c4e65abdcddcebd as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/436ae2d92cf643019c4e65abdcddcebd 2024-11-15T16:59:09,493 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/436ae2d92cf643019c4e65abdcddcebd, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T16:59:09,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/b2b5ff78502a4347887090df62ba0699 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/b2b5ff78502a4347887090df62ba0699 2024-11-15T16:59:09,498 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/b2b5ff78502a4347887090df62ba0699, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T16:59:09,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e122d8e8e1d3a6c89c4dc261c769f99f in 460ms, sequenceid=358, compaction requested=true 2024-11-15T16:59:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-15T16:59:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-15T16:59:09,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T16:59:09,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3810 sec 2024-11-15T16:59:09,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.3860 sec 2024-11-15T16:59:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:09,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:09,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T16:59:09,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/3e5c0792fbfd4842bee4644c63fbe772 is 50, key is test_row_0/A:col10/1731689949676/Put/seqid=0 2024-11-15T16:59:09,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742120_1296 (size=12301) 2024-11-15T16:59:09,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/3e5c0792fbfd4842bee4644c63fbe772 2024-11-15T16:59:09,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/266536a4cd9e462ba614b047007ca6b3 is 50, key is test_row_0/B:col10/1731689949676/Put/seqid=0 2024-11-15T16:59:09,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742121_1297 (size=12301) 2024-11-15T16:59:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690009768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690009876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690010085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:10,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/266536a4cd9e462ba614b047007ca6b3 2024-11-15T16:59:10,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d768d19b34504bdd850cd554c2e4375e is 50, key is test_row_0/C:col10/1731689949676/Put/seqid=0 2024-11-15T16:59:10,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742122_1298 (size=12301) 2024-11-15T16:59:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T16:59:10,266 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:10,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-15T16:59:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T16:59:10,269 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:10,269 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:10,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T16:59:10,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690010395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:10,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T16:59:10,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:10,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:10,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:10,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:10,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:10,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d768d19b34504bdd850cd554c2e4375e 2024-11-15T16:59:10,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/3e5c0792fbfd4842bee4644c63fbe772 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3e5c0792fbfd4842bee4644c63fbe772 2024-11-15T16:59:10,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3e5c0792fbfd4842bee4644c63fbe772, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T16:59:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/266536a4cd9e462ba614b047007ca6b3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/266536a4cd9e462ba614b047007ca6b3 2024-11-15T16:59:10,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/266536a4cd9e462ba614b047007ca6b3, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T16:59:10,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/d768d19b34504bdd850cd554c2e4375e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d768d19b34504bdd850cd554c2e4375e 2024-11-15T16:59:10,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d768d19b34504bdd850cd554c2e4375e, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T16:59:10,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e122d8e8e1d3a6c89c4dc261c769f99f in 854ms, sequenceid=370, compaction requested=true 2024-11-15T16:59:10,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:10,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:10,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:10,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:10,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:10,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:10,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:10,532 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:10,532 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:10,533 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:10,533 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:10,533 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:59:10,533 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:59:10,533 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:10,533 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e132ae8e2804965a2b69c004dc58d0f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/c72dead56252490b84e422655205d9a4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e6133b05a90441bd8bf4e803e39fb9ef, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3e5c0792fbfd4842bee4644c63fbe772] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=48.8 K 2024-11-15T16:59:10,534 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:10,534 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5bdd7b696f9e4ae29c14614b0aad907c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/df14dfa0767f46e4a6400e3a7f02107d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/436ae2d92cf643019c4e65abdcddcebd, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/266536a4cd9e462ba614b047007ca6b3] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=48.8 K 2024-11-15T16:59:10,534 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e132ae8e2804965a2b69c004dc58d0f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731689945564 2024-11-15T16:59:10,534 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bdd7b696f9e4ae29c14614b0aad907c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731689945564 2024-11-15T16:59:10,534 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c72dead56252490b84e422655205d9a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689948070 2024-11-15T16:59:10,535 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting df14dfa0767f46e4a6400e3a7f02107d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689948070 2024-11-15T16:59:10,535 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6133b05a90441bd8bf4e803e39fb9ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731689948148 2024-11-15T16:59:10,535 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 436ae2d92cf643019c4e65abdcddcebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731689948148 2024-11-15T16:59:10,535 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e5c0792fbfd4842bee4644c63fbe772, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689949354 2024-11-15T16:59:10,535 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 266536a4cd9e462ba614b047007ca6b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689949354 2024-11-15T16:59:10,544 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:10,545 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/02e3878c71ee4260a66b819a91db7818 is 50, key is test_row_0/B:col10/1731689949676/Put/seqid=0 2024-11-15T16:59:10,550 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:10,551 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/90516da23e74475f87e1949917ab0a8f is 50, key is test_row_0/A:col10/1731689949676/Put/seqid=0 2024-11-15T16:59:10,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742123_1299 (size=13187) 2024-11-15T16:59:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742124_1300 (size=13187) 2024-11-15T16:59:10,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T16:59:10,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:10,575 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T16:59:10,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/f2368a68b0fd41daa01c9f0932fd1bea is 50, key is test_row_0/A:col10/1731689949762/Put/seqid=0 2024-11-15T16:59:10,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742125_1301 (size=12301) 2024-11-15T16:59:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T16:59:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T16:59:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:10,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690010918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690010922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:10,961 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/02e3878c71ee4260a66b819a91db7818 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/02e3878c71ee4260a66b819a91db7818 2024-11-15T16:59:10,964 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/90516da23e74475f87e1949917ab0a8f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/90516da23e74475f87e1949917ab0a8f 2024-11-15T16:59:10,970 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into 02e3878c71ee4260a66b819a91db7818(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:10,970 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:10,970 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=12, startTime=1731689950532; duration=0sec 2024-11-15T16:59:10,970 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:10,970 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:59:10,970 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:10,972 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:10,972 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:59:10,972 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:10,972 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f4e46e648a4a46ba9a7dd461fe54c522, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2a3d4214cc4b4b8f84a5628d2c762d27, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/b2b5ff78502a4347887090df62ba0699, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d768d19b34504bdd850cd554c2e4375e] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=48.8 K 2024-11-15T16:59:10,972 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f4e46e648a4a46ba9a7dd461fe54c522, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731689945564 2024-11-15T16:59:10,973 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3d4214cc4b4b8f84a5628d2c762d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731689948070 2024-11-15T16:59:10,973 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b5ff78502a4347887090df62ba0699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731689948148 2024-11-15T16:59:10,973 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d768d19b34504bdd850cd554c2e4375e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689949354 2024-11-15T16:59:10,980 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 90516da23e74475f87e1949917ab0a8f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:10,980 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:10,980 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=12, startTime=1731689950531; duration=0sec 2024-11-15T16:59:10,980 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:10,980 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:59:10,981 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:10,982 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/c7a5f087871048f8862ddfe4878a2dfe is 50, key is test_row_0/C:col10/1731689949676/Put/seqid=0 2024-11-15T16:59:10,986 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/f2368a68b0fd41daa01c9f0932fd1bea 2024-11-15T16:59:10,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742126_1302 (size=13187) 2024-11-15T16:59:10,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/d917f859656048fabd168e4f66c6f97b is 50, key is test_row_0/B:col10/1731689949762/Put/seqid=0 2024-11-15T16:59:10,995 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/c7a5f087871048f8862ddfe4878a2dfe as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c7a5f087871048f8862ddfe4878a2dfe 2024-11-15T16:59:11,001 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into c7a5f087871048f8862ddfe4878a2dfe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:11,002 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:11,002 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=12, startTime=1731689950532; duration=0sec 2024-11-15T16:59:11,002 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:11,002 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:59:11,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742127_1303 (size=12301) 2024-11-15T16:59:11,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690011026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:11,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690011036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:11,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690011235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:11,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690011245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T16:59:11,406 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/d917f859656048fabd168e4f66c6f97b 2024-11-15T16:59:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/1a67ef2a446444658efb135f6f30ace6 is 50, key is test_row_0/C:col10/1731689949762/Put/seqid=0 2024-11-15T16:59:11,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742128_1304 (size=12301) 2024-11-15T16:59:11,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690011546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:11,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690011555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:11,819 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/1a67ef2a446444658efb135f6f30ace6 2024-11-15T16:59:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/f2368a68b0fd41daa01c9f0932fd1bea as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/f2368a68b0fd41daa01c9f0932fd1bea 2024-11-15T16:59:11,828 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/f2368a68b0fd41daa01c9f0932fd1bea, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T16:59:11,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/d917f859656048fabd168e4f66c6f97b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/d917f859656048fabd168e4f66c6f97b 2024-11-15T16:59:11,833 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/d917f859656048fabd168e4f66c6f97b, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T16:59:11,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/1a67ef2a446444658efb135f6f30ace6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/1a67ef2a446444658efb135f6f30ace6 2024-11-15T16:59:11,838 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/1a67ef2a446444658efb135f6f30ace6, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T16:59:11,838 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e122d8e8e1d3a6c89c4dc261c769f99f in 1263ms, sequenceid=394, compaction requested=false 2024-11-15T16:59:11,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:11,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:11,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-15T16:59:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-15T16:59:11,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-15T16:59:11,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-15T16:59:11,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.5770 sec 2024-11-15T16:59:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:12,057 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:12,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T16:59:12,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/d6df6c7874f64ca3b04484704258276d is 50, key is test_row_0/A:col10/1731689950912/Put/seqid=0 2024-11-15T16:59:12,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742129_1305 (size=12301) 2024-11-15T16:59:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690012103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690012106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690012215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690012216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48400 deadline: 1731690012226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,229 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731690012245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,248 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in e122d8e8e1d3a6c89c4dc261c769f99f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T16:59:12,416 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:12,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-15T16:59:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T16:59:12,418 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:12,419 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:12,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:12,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690012426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690012426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/d6df6c7874f64ca3b04484704258276d 2024-11-15T16:59:12,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5222c1f7a38945e7adfc0e1ec3671d97 is 50, key is test_row_0/B:col10/1731689950912/Put/seqid=0 2024-11-15T16:59:12,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742130_1306 (size=12301) 2024-11-15T16:59:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T16:59:12,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-15T16:59:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:12,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-15T16:59:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:12,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T16:59:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690012735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690012735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:12,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-15T16:59:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:12,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:12,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5222c1f7a38945e7adfc0e1ec3671d97 2024-11-15T16:59:12,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/c4fc77a58a774d28961a9163f08bfd9c is 50, key is test_row_0/C:col10/1731689950912/Put/seqid=0 2024-11-15T16:59:12,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742131_1307 (size=12301) 2024-11-15T16:59:12,927 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-11-15T16:59:12,927 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:12,927 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,928 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:12,928 INFO [Thread-1051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1051. 2024-11-15T16:59:12,928 DEBUG [Thread-1051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:12,928 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,928 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:12,929 INFO [Thread-1049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1049. 2024-11-15T16:59:12,929 DEBUG [Thread-1049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:12,929 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,930 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:12,931 INFO [Thread-1055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1055. 2024-11-15T16:59:12,931 DEBUG [Thread-1055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:12,931 DEBUG [Thread-1055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,931 DEBUG [Thread-1055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:12,931 INFO [Thread-1053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1053. 2024-11-15T16:59:12,931 DEBUG [Thread-1053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:12,932 DEBUG [Thread-1053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,932 DEBUG [Thread-1053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:12,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:13,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-15T16:59:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T16:59:13,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-15T16:59:13,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:13,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:13,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:13,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:13,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48434 deadline: 1731690013245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48410 deadline: 1731690013245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:13,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/c4fc77a58a774d28961a9163f08bfd9c 2024-11-15T16:59:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/d6df6c7874f64ca3b04484704258276d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d6df6c7874f64ca3b04484704258276d 2024-11-15T16:59:13,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d6df6c7874f64ca3b04484704258276d, entries=150, sequenceid=410, filesize=12.0 K 2024-11-15T16:59:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/5222c1f7a38945e7adfc0e1ec3671d97 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5222c1f7a38945e7adfc0e1ec3671d97 2024-11-15T16:59:13,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5222c1f7a38945e7adfc0e1ec3671d97, entries=150, sequenceid=410, filesize=12.0 K 2024-11-15T16:59:13,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/c4fc77a58a774d28961a9163f08bfd9c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c4fc77a58a774d28961a9163f08bfd9c 2024-11-15T16:59:13,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c4fc77a58a774d28961a9163f08bfd9c, entries=150, sequenceid=410, filesize=12.0 K 2024-11-15T16:59:13,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e122d8e8e1d3a6c89c4dc261c769f99f in 1250ms, sequenceid=410, compaction requested=true 2024-11-15T16:59:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:13,308 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e122d8e8e1d3a6c89c4dc261c769f99f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:13,308 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:13,309 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:13,309 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:13,309 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/A is initiating minor compaction (all files) 2024-11-15T16:59:13,309 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/B is initiating minor compaction (all files) 2024-11-15T16:59:13,309 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/B in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,309 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/A in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,309 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/02e3878c71ee4260a66b819a91db7818, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/d917f859656048fabd168e4f66c6f97b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5222c1f7a38945e7adfc0e1ec3671d97] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=36.9 K 2024-11-15T16:59:13,309 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/90516da23e74475f87e1949917ab0a8f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/f2368a68b0fd41daa01c9f0932fd1bea, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d6df6c7874f64ca3b04484704258276d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=36.9 K 2024-11-15T16:59:13,309 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 02e3878c71ee4260a66b819a91db7818, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689949354 2024-11-15T16:59:13,309 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90516da23e74475f87e1949917ab0a8f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689949354 2024-11-15T16:59:13,310 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2368a68b0fd41daa01c9f0932fd1bea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731689949757 2024-11-15T16:59:13,310 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d917f859656048fabd168e4f66c6f97b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731689949757 2024-11-15T16:59:13,310 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6df6c7874f64ca3b04484704258276d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731689950912 2024-11-15T16:59:13,310 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5222c1f7a38945e7adfc0e1ec3671d97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731689950912 2024-11-15T16:59:13,316 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:13,316 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:13,317 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/3148742918494babb94edf90634bb764 is 50, key is test_row_0/A:col10/1731689950912/Put/seqid=0 2024-11-15T16:59:13,317 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/695ff3f5f9b943499af4a18f2752de6d is 50, key is test_row_0/B:col10/1731689950912/Put/seqid=0 2024-11-15T16:59:13,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742133_1309 (size=13289) 2024-11-15T16:59:13,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742132_1308 (size=13289) 2024-11-15T16:59:13,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-15T16:59:13,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,338 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T16:59:13,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/6da5590170f646f59b6478ec66ad5bc6 is 50, key is test_row_0/A:col10/1731689952105/Put/seqid=0 2024-11-15T16:59:13,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742134_1310 (size=12301) 2024-11-15T16:59:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T16:59:13,725 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/695ff3f5f9b943499af4a18f2752de6d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/695ff3f5f9b943499af4a18f2752de6d 2024-11-15T16:59:13,725 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/3148742918494babb94edf90634bb764 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3148742918494babb94edf90634bb764 2024-11-15T16:59:13,729 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/B of e122d8e8e1d3a6c89c4dc261c769f99f into 695ff3f5f9b943499af4a18f2752de6d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:13,729 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/A of e122d8e8e1d3a6c89c4dc261c769f99f into 3148742918494babb94edf90634bb764(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:13,729 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:13,729 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:13,729 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/A, priority=13, startTime=1731689953308; duration=0sec 2024-11-15T16:59:13,729 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/B, priority=13, startTime=1731689953308; duration=0sec 2024-11-15T16:59:13,729 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:13,729 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:A 2024-11-15T16:59:13,729 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:13,729 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:B 2024-11-15T16:59:13,729 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:13,730 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:13,730 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): e122d8e8e1d3a6c89c4dc261c769f99f/C is initiating minor compaction (all files) 2024-11-15T16:59:13,730 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e122d8e8e1d3a6c89c4dc261c769f99f/C in TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:13,730 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c7a5f087871048f8862ddfe4878a2dfe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/1a67ef2a446444658efb135f6f30ace6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c4fc77a58a774d28961a9163f08bfd9c] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp, totalSize=36.9 K 2024-11-15T16:59:13,730 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7a5f087871048f8862ddfe4878a2dfe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731689949354 2024-11-15T16:59:13,730 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a67ef2a446444658efb135f6f30ace6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731689949757 2024-11-15T16:59:13,731 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4fc77a58a774d28961a9163f08bfd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731689950912 2024-11-15T16:59:13,736 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e122d8e8e1d3a6c89c4dc261c769f99f#C#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:13,736 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/9488360e12764ee5b53756ecf70dbb05 is 50, key is test_row_0/C:col10/1731689950912/Put/seqid=0 2024-11-15T16:59:13,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742135_1311 (size=13289) 2024-11-15T16:59:13,745 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/6da5590170f646f59b6478ec66ad5bc6 2024-11-15T16:59:13,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/29eb173599a74225a88e74ffd2a27926 is 50, key is test_row_0/B:col10/1731689952105/Put/seqid=0 2024-11-15T16:59:13,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742136_1312 (size=12301) 2024-11-15T16:59:14,144 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/9488360e12764ee5b53756ecf70dbb05 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9488360e12764ee5b53756ecf70dbb05 2024-11-15T16:59:14,149 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e122d8e8e1d3a6c89c4dc261c769f99f/C of e122d8e8e1d3a6c89c4dc261c769f99f into 9488360e12764ee5b53756ecf70dbb05(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:14,149 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:14,149 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., storeName=e122d8e8e1d3a6c89c4dc261c769f99f/C, priority=13, startTime=1731689953308; duration=0sec 2024-11-15T16:59:14,149 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:14,149 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e122d8e8e1d3a6c89c4dc261c769f99f:C 2024-11-15T16:59:14,154 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/29eb173599a74225a88e74ffd2a27926 2024-11-15T16:59:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/8cd5b2ff5b4e4d4da535cd752db757fa is 50, key is test_row_0/C:col10/1731689952105/Put/seqid=0 2024-11-15T16:59:14,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742137_1313 (size=12301) 2024-11-15T16:59:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:14,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. as already flushing 2024-11-15T16:59:14,257 INFO [Thread-1042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1042. 2024-11-15T16:59:14,257 INFO [Thread-1038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1038. 2024-11-15T16:59:14,257 DEBUG [Thread-1042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:14,257 DEBUG [Thread-1038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:14,257 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:14,257 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:14,257 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:14,257 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:14,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:14,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:14,563 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/8cd5b2ff5b4e4d4da535cd752db757fa 2024-11-15T16:59:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T16:59:14,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/6da5590170f646f59b6478ec66ad5bc6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6da5590170f646f59b6478ec66ad5bc6 2024-11-15T16:59:14,571 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6da5590170f646f59b6478ec66ad5bc6, entries=150, sequenceid=433, filesize=12.0 K 2024-11-15T16:59:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/29eb173599a74225a88e74ffd2a27926 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/29eb173599a74225a88e74ffd2a27926 2024-11-15T16:59:14,575 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/29eb173599a74225a88e74ffd2a27926, entries=150, sequenceid=433, filesize=12.0 K 2024-11-15T16:59:14,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/8cd5b2ff5b4e4d4da535cd752db757fa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/8cd5b2ff5b4e4d4da535cd752db757fa 2024-11-15T16:59:14,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/8cd5b2ff5b4e4d4da535cd752db757fa, entries=150, sequenceid=433, filesize=12.0 K 2024-11-15T16:59:14,579 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=13.42 KB/13740 for e122d8e8e1d3a6c89c4dc261c769f99f in 1241ms, sequenceid=433, compaction requested=false 2024-11-15T16:59:14,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for e122d8e8e1d3a6c89c4dc261c769f99f: 2024-11-15T16:59:14,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:14,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-15T16:59:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-15T16:59:14,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T16:59:14,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-15T16:59:14,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 2.1660 sec 2024-11-15T16:59:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T16:59:16,596 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:17,147 INFO [Thread-1036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1036. 2024-11-15T16:59:17,147 DEBUG [Thread-1036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:17,147 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:17,147 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:17,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:20,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T16:59:22,258 INFO [Thread-1044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1044. 2024-11-15T16:59:22,258 DEBUG [Thread-1044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:22,258 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:22,258 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:22,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:22,287 INFO [Thread-1040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1040. 2024-11-15T16:59:22,287 DEBUG [Thread-1040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:22,287 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:22,287 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:22,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4064 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4144 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4177 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4072 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4142 2024-11-15T16:59:22,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T16:59:22,288 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:59:22,288 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:59:22,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:22,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:22,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:22,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T16:59:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T16:59:22,291 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689962291"}]},"ts":"1731689962291"} 2024-11-15T16:59:22,292 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T16:59:22,293 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T16:59:22,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T16:59:22,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, UNASSIGN}] 2024-11-15T16:59:22,295 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, UNASSIGN 2024-11-15T16:59:22,295 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e122d8e8e1d3a6c89c4dc261c769f99f, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:22,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, UNASSIGN because future has completed 2024-11-15T16:59:22,297 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:59:22,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:59:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T16:59:22,449 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:22,449 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:59:22,449 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing e122d8e8e1d3a6c89c4dc261c769f99f, disabling compactions & flushes 2024-11-15T16:59:22,449 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:22,449 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:22,449 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. after waiting 0 ms 2024-11-15T16:59:22,449 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:22,449 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing e122d8e8e1d3a6c89c4dc261c769f99f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T16:59:22,453 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/d3ab0210e0164b78977764f1ff9b80e2 is 50, key is test_row_1/A:col10/1731689962286/Put/seqid=0 2024-11-15T16:59:22,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742138_1314 (size=7415) 2024-11-15T16:59:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T16:59:22,857 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/d3ab0210e0164b78977764f1ff9b80e2 2024-11-15T16:59:22,863 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/e42b027f44c4412cb9a515dc60cf5800 is 50, key is test_row_1/B:col10/1731689962286/Put/seqid=0 2024-11-15T16:59:22,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742139_1315 (size=7415) 2024-11-15T16:59:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T16:59:23,267 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/e42b027f44c4412cb9a515dc60cf5800 2024-11-15T16:59:23,273 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/5fe41d97aca4464f8c0a398661c9e62e is 50, key is test_row_1/C:col10/1731689962286/Put/seqid=0 2024-11-15T16:59:23,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742140_1316 (size=7415) 2024-11-15T16:59:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T16:59:23,677 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/5fe41d97aca4464f8c0a398661c9e62e 2024-11-15T16:59:23,681 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/A/d3ab0210e0164b78977764f1ff9b80e2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d3ab0210e0164b78977764f1ff9b80e2 2024-11-15T16:59:23,684 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d3ab0210e0164b78977764f1ff9b80e2, entries=50, sequenceid=444, filesize=7.2 K 2024-11-15T16:59:23,685 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/B/e42b027f44c4412cb9a515dc60cf5800 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e42b027f44c4412cb9a515dc60cf5800 2024-11-15T16:59:23,688 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e42b027f44c4412cb9a515dc60cf5800, entries=50, sequenceid=444, filesize=7.2 K 2024-11-15T16:59:23,688 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/.tmp/C/5fe41d97aca4464f8c0a398661c9e62e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/5fe41d97aca4464f8c0a398661c9e62e 2024-11-15T16:59:23,691 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/5fe41d97aca4464f8c0a398661c9e62e, entries=50, sequenceid=444, filesize=7.2 K 2024-11-15T16:59:23,692 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e122d8e8e1d3a6c89c4dc261c769f99f in 1243ms, sequenceid=444, compaction requested=true 2024-11-15T16:59:23,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9812f21204064146a1a86798c8ae68a3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/a872eefb4c7a4645abbc8c0f7444beeb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cb22e4eae9c54b20a11654b4057b757f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/dccd1abbf1a4471ab1294731594c5464, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/111838aad37b4537a0008ab4431aad61, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/4aae7bd1d29f4319be95fa80c5b0cf26, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/78cb1caf353b494e95e9e907a948e3bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6f7fa71f0e0442d68db586680cb5ebc6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/5878ae0657804be0927dae4e651c4925, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/35962ad5577746f095380be78dd5ffe5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/8c5ab64f08d64115a7d940e0d91dd241, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e970b289fd3540599e7c517192e20f4a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/340c64aea5954c418d5cc02b137a561b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/38b140857d18485ba3d2cecb8fca5418, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/2268ecf0dd70433681edef8300e109e6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/7283825b2db54243b9aa6ab772c3e69c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e662a87fa944aa5aca89cfc1880a13a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9bf6ecbaaaf34793a11bbdd213dc3410, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cf2f9988aa1c4567a15b285173d44684, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/14c29f83acd94ca9a52968e8a1dade5f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/34303a21cc6b45a7b0a85e24624c4343, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/792ad495b4924786869d17baf67ddc81, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e132ae8e2804965a2b69c004dc58d0f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/c72dead56252490b84e422655205d9a4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e6133b05a90441bd8bf4e803e39fb9ef, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/90516da23e74475f87e1949917ab0a8f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3e5c0792fbfd4842bee4644c63fbe772, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/f2368a68b0fd41daa01c9f0932fd1bea, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d6df6c7874f64ca3b04484704258276d] to archive 2024-11-15T16:59:23,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:59:23,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9812f21204064146a1a86798c8ae68a3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9812f21204064146a1a86798c8ae68a3 2024-11-15T16:59:23,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/a872eefb4c7a4645abbc8c0f7444beeb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/a872eefb4c7a4645abbc8c0f7444beeb 2024-11-15T16:59:23,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cb22e4eae9c54b20a11654b4057b757f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cb22e4eae9c54b20a11654b4057b757f 2024-11-15T16:59:23,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/dccd1abbf1a4471ab1294731594c5464 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/dccd1abbf1a4471ab1294731594c5464 2024-11-15T16:59:23,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/111838aad37b4537a0008ab4431aad61 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/111838aad37b4537a0008ab4431aad61 2024-11-15T16:59:23,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/4aae7bd1d29f4319be95fa80c5b0cf26 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/4aae7bd1d29f4319be95fa80c5b0cf26 2024-11-15T16:59:23,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/78cb1caf353b494e95e9e907a948e3bb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/78cb1caf353b494e95e9e907a948e3bb 2024-11-15T16:59:23,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6f7fa71f0e0442d68db586680cb5ebc6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6f7fa71f0e0442d68db586680cb5ebc6 2024-11-15T16:59:23,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/5878ae0657804be0927dae4e651c4925 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/5878ae0657804be0927dae4e651c4925 2024-11-15T16:59:23,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/35962ad5577746f095380be78dd5ffe5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/35962ad5577746f095380be78dd5ffe5 2024-11-15T16:59:23,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/8c5ab64f08d64115a7d940e0d91dd241 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/8c5ab64f08d64115a7d940e0d91dd241 2024-11-15T16:59:23,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e970b289fd3540599e7c517192e20f4a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e970b289fd3540599e7c517192e20f4a 2024-11-15T16:59:23,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/340c64aea5954c418d5cc02b137a561b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/340c64aea5954c418d5cc02b137a561b 2024-11-15T16:59:23,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/38b140857d18485ba3d2cecb8fca5418 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/38b140857d18485ba3d2cecb8fca5418 2024-11-15T16:59:23,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/2268ecf0dd70433681edef8300e109e6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/2268ecf0dd70433681edef8300e109e6 2024-11-15T16:59:23,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/7283825b2db54243b9aa6ab772c3e69c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/7283825b2db54243b9aa6ab772c3e69c 2024-11-15T16:59:23,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e662a87fa944aa5aca89cfc1880a13a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e662a87fa944aa5aca89cfc1880a13a 2024-11-15T16:59:23,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9bf6ecbaaaf34793a11bbdd213dc3410 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/9bf6ecbaaaf34793a11bbdd213dc3410 2024-11-15T16:59:23,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cf2f9988aa1c4567a15b285173d44684 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/cf2f9988aa1c4567a15b285173d44684 2024-11-15T16:59:23,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/14c29f83acd94ca9a52968e8a1dade5f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/14c29f83acd94ca9a52968e8a1dade5f 2024-11-15T16:59:23,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/34303a21cc6b45a7b0a85e24624c4343 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/34303a21cc6b45a7b0a85e24624c4343 2024-11-15T16:59:23,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/792ad495b4924786869d17baf67ddc81 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/792ad495b4924786869d17baf67ddc81 2024-11-15T16:59:23,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e132ae8e2804965a2b69c004dc58d0f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/1e132ae8e2804965a2b69c004dc58d0f 2024-11-15T16:59:23,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/c72dead56252490b84e422655205d9a4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/c72dead56252490b84e422655205d9a4 2024-11-15T16:59:23,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e6133b05a90441bd8bf4e803e39fb9ef to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/e6133b05a90441bd8bf4e803e39fb9ef 2024-11-15T16:59:23,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/90516da23e74475f87e1949917ab0a8f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/90516da23e74475f87e1949917ab0a8f 2024-11-15T16:59:23,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3e5c0792fbfd4842bee4644c63fbe772 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3e5c0792fbfd4842bee4644c63fbe772 2024-11-15T16:59:23,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/f2368a68b0fd41daa01c9f0932fd1bea to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/f2368a68b0fd41daa01c9f0932fd1bea 2024-11-15T16:59:23,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d6df6c7874f64ca3b04484704258276d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d6df6c7874f64ca3b04484704258276d 2024-11-15T16:59:23,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/bd15ebca5641476e970d2814ad8eaa45, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6a7d048dd1aa48f38b7539623ce90519, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/2d94c79b02c04cc9a675422913b04b01, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/f6b6994de6c247bcaeb6624e7b698494, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6bc8b0be7358481eba609cce1530aa3c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cca2c5b9952b40a4a7a70194daf043ee, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/823b7b316f9a428f969a765f3be1aa7e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/b85b2e433d8e499ebe0760df59f4c5d3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5ce564feb6844213b3a5152108e5b51b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e30e37ecf7fe42c298f007baa69f3d76, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/9ee0952a2825460c8124f43fab64ce09, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d13fd7c478c438080b1ff36ce4b96b0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/63a1f207180a4fec9ca5956cd3f8b2a4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6f596990ee914c18b23d7ca1d988be04, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/988444757b364ca5843adf4e6a608733, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cfe015a1d1ec403ab523587da2ba6c33, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/36451dbcfa6a498b9b146d84b0648de0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/525acfdc8bb340669078b9fa5c4a1f00, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/332d210532bf4e71a379595ddf5a8148, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/04449fcdf3fb4bff8a69da19705e6dd4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d788cc649a44fe5baed929e21042e56, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5bdd7b696f9e4ae29c14614b0aad907c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/8086dd4bf52649569fc43c1fe2fd65c4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/df14dfa0767f46e4a6400e3a7f02107d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/436ae2d92cf643019c4e65abdcddcebd, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/02e3878c71ee4260a66b819a91db7818, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/266536a4cd9e462ba614b047007ca6b3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/d917f859656048fabd168e4f66c6f97b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5222c1f7a38945e7adfc0e1ec3671d97] to archive 2024-11-15T16:59:23,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:59:23,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/bd15ebca5641476e970d2814ad8eaa45 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/bd15ebca5641476e970d2814ad8eaa45 2024-11-15T16:59:23,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6a7d048dd1aa48f38b7539623ce90519 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6a7d048dd1aa48f38b7539623ce90519 2024-11-15T16:59:23,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/2d94c79b02c04cc9a675422913b04b01 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/2d94c79b02c04cc9a675422913b04b01 2024-11-15T16:59:23,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/f6b6994de6c247bcaeb6624e7b698494 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/f6b6994de6c247bcaeb6624e7b698494 2024-11-15T16:59:23,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6bc8b0be7358481eba609cce1530aa3c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6bc8b0be7358481eba609cce1530aa3c 2024-11-15T16:59:23,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cca2c5b9952b40a4a7a70194daf043ee to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cca2c5b9952b40a4a7a70194daf043ee 2024-11-15T16:59:23,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/823b7b316f9a428f969a765f3be1aa7e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/823b7b316f9a428f969a765f3be1aa7e 2024-11-15T16:59:23,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/b85b2e433d8e499ebe0760df59f4c5d3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/b85b2e433d8e499ebe0760df59f4c5d3 2024-11-15T16:59:23,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5ce564feb6844213b3a5152108e5b51b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5ce564feb6844213b3a5152108e5b51b 2024-11-15T16:59:23,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e30e37ecf7fe42c298f007baa69f3d76 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e30e37ecf7fe42c298f007baa69f3d76 2024-11-15T16:59:23,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/9ee0952a2825460c8124f43fab64ce09 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/9ee0952a2825460c8124f43fab64ce09 2024-11-15T16:59:23,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d13fd7c478c438080b1ff36ce4b96b0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d13fd7c478c438080b1ff36ce4b96b0 2024-11-15T16:59:23,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/63a1f207180a4fec9ca5956cd3f8b2a4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/63a1f207180a4fec9ca5956cd3f8b2a4 2024-11-15T16:59:23,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6f596990ee914c18b23d7ca1d988be04 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/6f596990ee914c18b23d7ca1d988be04 2024-11-15T16:59:23,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/988444757b364ca5843adf4e6a608733 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/988444757b364ca5843adf4e6a608733 2024-11-15T16:59:23,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cfe015a1d1ec403ab523587da2ba6c33 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/cfe015a1d1ec403ab523587da2ba6c33 2024-11-15T16:59:23,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/36451dbcfa6a498b9b146d84b0648de0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/36451dbcfa6a498b9b146d84b0648de0 2024-11-15T16:59:23,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/525acfdc8bb340669078b9fa5c4a1f00 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/525acfdc8bb340669078b9fa5c4a1f00 2024-11-15T16:59:23,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/332d210532bf4e71a379595ddf5a8148 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/332d210532bf4e71a379595ddf5a8148 2024-11-15T16:59:23,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/04449fcdf3fb4bff8a69da19705e6dd4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/04449fcdf3fb4bff8a69da19705e6dd4 2024-11-15T16:59:23,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d788cc649a44fe5baed929e21042e56 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/1d788cc649a44fe5baed929e21042e56 2024-11-15T16:59:23,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5bdd7b696f9e4ae29c14614b0aad907c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5bdd7b696f9e4ae29c14614b0aad907c 2024-11-15T16:59:23,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/8086dd4bf52649569fc43c1fe2fd65c4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/8086dd4bf52649569fc43c1fe2fd65c4 2024-11-15T16:59:23,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/df14dfa0767f46e4a6400e3a7f02107d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/df14dfa0767f46e4a6400e3a7f02107d 2024-11-15T16:59:23,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/436ae2d92cf643019c4e65abdcddcebd to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/436ae2d92cf643019c4e65abdcddcebd 2024-11-15T16:59:23,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/02e3878c71ee4260a66b819a91db7818 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/02e3878c71ee4260a66b819a91db7818 2024-11-15T16:59:23,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/266536a4cd9e462ba614b047007ca6b3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/266536a4cd9e462ba614b047007ca6b3 2024-11-15T16:59:23,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/d917f859656048fabd168e4f66c6f97b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/d917f859656048fabd168e4f66c6f97b 2024-11-15T16:59:23,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5222c1f7a38945e7adfc0e1ec3671d97 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/5222c1f7a38945e7adfc0e1ec3671d97 2024-11-15T16:59:23,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/17fc4c0b849d42e2949bbeb7370377bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/bf8534ce5dcb4829bd244d7b1543756e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/cbce6563b58f4d8f8194395de34e1019, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e52577a538e84a9781e6df5f9218719a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/7b092f4892e7486da9f3eaa434ff9568, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/907ff38cd9f44c77908c34d501a0baf1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/237a17164453419c8ee5c688f48ae7bb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/43e4245998c54526a34c1c941f75d90c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/6cdf4e31b19740fc8b47dc56fe01b8d6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/20c72ab3daf043e0bf19ea01f0b0bf03, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2ecbe1fbe44c4ed182582ae4c4b2f04a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d64fab27e73b44c8a8b5f7f548bd6e63, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9671758b64de41fda5a47057be01843d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/03afb06361ad4228a58def5fd77f4f95, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f44749c167e4406889a595b438e97972, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d6edb73dcc9f4a9b898e2793b50077c7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/446785c65d2a44ad9ab311cfd1cac930, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/227a4a0c9bc042d881a2384941fb4b7a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/18b65f426323424483d9afd657bff24a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e72032b4255f4837881bd3681ddd299d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/065137dd3733472ca48241d2ae293ea4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f4e46e648a4a46ba9a7dd461fe54c522, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/110573bf16e24c4985830a078c32cfad, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2a3d4214cc4b4b8f84a5628d2c762d27, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/b2b5ff78502a4347887090df62ba0699, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c7a5f087871048f8862ddfe4878a2dfe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d768d19b34504bdd850cd554c2e4375e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/1a67ef2a446444658efb135f6f30ace6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c4fc77a58a774d28961a9163f08bfd9c] to archive 2024-11-15T16:59:23,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:59:23,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/17fc4c0b849d42e2949bbeb7370377bb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/17fc4c0b849d42e2949bbeb7370377bb 2024-11-15T16:59:23,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/bf8534ce5dcb4829bd244d7b1543756e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/bf8534ce5dcb4829bd244d7b1543756e 2024-11-15T16:59:23,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/cbce6563b58f4d8f8194395de34e1019 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/cbce6563b58f4d8f8194395de34e1019 2024-11-15T16:59:23,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e52577a538e84a9781e6df5f9218719a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e52577a538e84a9781e6df5f9218719a 2024-11-15T16:59:23,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/7b092f4892e7486da9f3eaa434ff9568 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/7b092f4892e7486da9f3eaa434ff9568 2024-11-15T16:59:23,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/907ff38cd9f44c77908c34d501a0baf1 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/907ff38cd9f44c77908c34d501a0baf1 2024-11-15T16:59:23,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/237a17164453419c8ee5c688f48ae7bb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/237a17164453419c8ee5c688f48ae7bb 2024-11-15T16:59:23,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/43e4245998c54526a34c1c941f75d90c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/43e4245998c54526a34c1c941f75d90c 2024-11-15T16:59:23,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/6cdf4e31b19740fc8b47dc56fe01b8d6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/6cdf4e31b19740fc8b47dc56fe01b8d6 2024-11-15T16:59:23,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/20c72ab3daf043e0bf19ea01f0b0bf03 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/20c72ab3daf043e0bf19ea01f0b0bf03 2024-11-15T16:59:23,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2ecbe1fbe44c4ed182582ae4c4b2f04a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2ecbe1fbe44c4ed182582ae4c4b2f04a 2024-11-15T16:59:23,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d64fab27e73b44c8a8b5f7f548bd6e63 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d64fab27e73b44c8a8b5f7f548bd6e63 2024-11-15T16:59:23,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9671758b64de41fda5a47057be01843d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9671758b64de41fda5a47057be01843d 2024-11-15T16:59:23,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/03afb06361ad4228a58def5fd77f4f95 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/03afb06361ad4228a58def5fd77f4f95 2024-11-15T16:59:23,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f44749c167e4406889a595b438e97972 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f44749c167e4406889a595b438e97972 2024-11-15T16:59:23,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d6edb73dcc9f4a9b898e2793b50077c7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d6edb73dcc9f4a9b898e2793b50077c7 2024-11-15T16:59:23,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/446785c65d2a44ad9ab311cfd1cac930 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/446785c65d2a44ad9ab311cfd1cac930 2024-11-15T16:59:23,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/227a4a0c9bc042d881a2384941fb4b7a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/227a4a0c9bc042d881a2384941fb4b7a 2024-11-15T16:59:23,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/18b65f426323424483d9afd657bff24a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/18b65f426323424483d9afd657bff24a 2024-11-15T16:59:23,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e72032b4255f4837881bd3681ddd299d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/e72032b4255f4837881bd3681ddd299d 2024-11-15T16:59:23,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/065137dd3733472ca48241d2ae293ea4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/065137dd3733472ca48241d2ae293ea4 2024-11-15T16:59:23,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f4e46e648a4a46ba9a7dd461fe54c522 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/f4e46e648a4a46ba9a7dd461fe54c522 2024-11-15T16:59:23,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/110573bf16e24c4985830a078c32cfad to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/110573bf16e24c4985830a078c32cfad 2024-11-15T16:59:23,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2a3d4214cc4b4b8f84a5628d2c762d27 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/2a3d4214cc4b4b8f84a5628d2c762d27 2024-11-15T16:59:23,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/b2b5ff78502a4347887090df62ba0699 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/b2b5ff78502a4347887090df62ba0699 2024-11-15T16:59:23,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c7a5f087871048f8862ddfe4878a2dfe to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c7a5f087871048f8862ddfe4878a2dfe 2024-11-15T16:59:23,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d768d19b34504bdd850cd554c2e4375e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/d768d19b34504bdd850cd554c2e4375e 2024-11-15T16:59:23,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/1a67ef2a446444658efb135f6f30ace6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/1a67ef2a446444658efb135f6f30ace6 2024-11-15T16:59:23,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c4fc77a58a774d28961a9163f08bfd9c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/c4fc77a58a774d28961a9163f08bfd9c 2024-11-15T16:59:23,789 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/recovered.edits/447.seqid, newMaxSeqId=447, maxSeqId=1 2024-11-15T16:59:23,790 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. 2024-11-15T16:59:23,790 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for e122d8e8e1d3a6c89c4dc261c769f99f: Waiting for close lock at 1731689962449Running coprocessor pre-close hooks at 1731689962449Disabling compacts and flushes for region at 1731689962449Disabling writes for close at 1731689962449Obtaining lock to block concurrent updates at 1731689962449Preparing flush snapshotting stores in e122d8e8e1d3a6c89c4dc261c769f99f at 1731689962449Finished memstore snapshotting TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731689962449Flushing stores of TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f. at 1731689962450 (+1 ms)Flushing e122d8e8e1d3a6c89c4dc261c769f99f/A: creating writer at 1731689962450Flushing e122d8e8e1d3a6c89c4dc261c769f99f/A: appending metadata at 1731689962452 (+2 ms)Flushing e122d8e8e1d3a6c89c4dc261c769f99f/A: closing flushed file at 1731689962452Flushing e122d8e8e1d3a6c89c4dc261c769f99f/B: creating writer at 1731689962861 (+409 ms)Flushing e122d8e8e1d3a6c89c4dc261c769f99f/B: appending metadata at 1731689962863 (+2 ms)Flushing e122d8e8e1d3a6c89c4dc261c769f99f/B: closing flushed file at 1731689962863Flushing e122d8e8e1d3a6c89c4dc261c769f99f/C: creating writer at 1731689963271 (+408 ms)Flushing e122d8e8e1d3a6c89c4dc261c769f99f/C: appending metadata at 1731689963273 (+2 ms)Flushing e122d8e8e1d3a6c89c4dc261c769f99f/C: closing flushed file at 1731689963273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5838fa87: reopening flushed file at 1731689963681 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46f6e510: reopening flushed file at 1731689963684 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d4bb67d: reopening flushed file at 1731689963688 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e122d8e8e1d3a6c89c4dc261c769f99f in 1243ms, sequenceid=444, compaction requested=true at 1731689963692 (+4 ms)Writing region close event to WAL at 1731689963786 (+94 ms)Running coprocessor post-close hooks at 1731689963790 (+4 ms)Closed at 1731689963790 2024-11-15T16:59:23,792 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:23,793 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e122d8e8e1d3a6c89c4dc261c769f99f, regionState=CLOSED 2024-11-15T16:59:23,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:59:23,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-15T16:59:23,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure e122d8e8e1d3a6c89c4dc261c769f99f, server=a9bde5b3a1b5,35119,1731689872090 in 1.4980 sec 2024-11-15T16:59:23,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-15T16:59:23,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e122d8e8e1d3a6c89c4dc261c769f99f, UNASSIGN in 1.5030 sec 2024-11-15T16:59:23,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-15T16:59:23,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5060 sec 2024-11-15T16:59:23,802 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689963802"}]},"ts":"1731689963802"} 2024-11-15T16:59:23,803 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T16:59:23,803 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T16:59:23,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-15T16:59:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T16:59:24,446 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:24,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T16:59:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:24,448 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-15T16:59:24,449 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:24,451 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:24,452 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/recovered.edits] 2024-11-15T16:59:24,455 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3148742918494babb94edf90634bb764 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/3148742918494babb94edf90634bb764 2024-11-15T16:59:24,455 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6da5590170f646f59b6478ec66ad5bc6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/6da5590170f646f59b6478ec66ad5bc6 2024-11-15T16:59:24,457 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d3ab0210e0164b78977764f1ff9b80e2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/A/d3ab0210e0164b78977764f1ff9b80e2 2024-11-15T16:59:24,459 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/29eb173599a74225a88e74ffd2a27926 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/29eb173599a74225a88e74ffd2a27926 2024-11-15T16:59:24,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/695ff3f5f9b943499af4a18f2752de6d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/695ff3f5f9b943499af4a18f2752de6d 2024-11-15T16:59:24,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e42b027f44c4412cb9a515dc60cf5800 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/B/e42b027f44c4412cb9a515dc60cf5800 2024-11-15T16:59:24,462 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/5fe41d97aca4464f8c0a398661c9e62e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/5fe41d97aca4464f8c0a398661c9e62e 2024-11-15T16:59:24,463 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/8cd5b2ff5b4e4d4da535cd752db757fa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/8cd5b2ff5b4e4d4da535cd752db757fa 2024-11-15T16:59:24,464 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9488360e12764ee5b53756ecf70dbb05 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/C/9488360e12764ee5b53756ecf70dbb05 2024-11-15T16:59:24,466 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/recovered.edits/447.seqid to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f/recovered.edits/447.seqid 2024-11-15T16:59:24,467 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/e122d8e8e1d3a6c89c4dc261c769f99f 2024-11-15T16:59:24,467 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T16:59:24,469 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:24,472 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T16:59:24,474 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T16:59:24,475 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:24,475 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T16:59:24,475 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731689964475"}]},"ts":"9223372036854775807"} 2024-11-15T16:59:24,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T16:59:24,477 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e122d8e8e1d3a6c89c4dc261c769f99f, NAME => 'TestAcidGuarantees,,1731689930735.e122d8e8e1d3a6c89c4dc261c769f99f.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T16:59:24,477 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T16:59:24,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731689964477"}]},"ts":"9223372036854775807"} 2024-11-15T16:59:24,479 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T16:59:24,479 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:24,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-15T16:59:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-15T16:59:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T16:59:24,556 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:24,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 240), OpenFileDescriptor=445 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=311 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=5426 (was 5517) 2024-11-15T16:59:24,575 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=5426 2024-11-15T16:59:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T16:59:24,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:59:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:24,579 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:59:24,579 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:24,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-15T16:59:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T16:59:24,580 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:59:24,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742141_1317 (size=959) 2024-11-15T16:59:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T16:59:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T16:59:24,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T16:59:24,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742142_1318 (size=53) 2024-11-15T16:59:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T16:59:25,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:59:25,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fec8d5aa95a6ec0fca101912a0d79069, disabling compactions & flushes 2024-11-15T16:59:25,392 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:25,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:25,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. after waiting 0 ms 2024-11-15T16:59:25,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:25,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:25,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fec8d5aa95a6ec0fca101912a0d79069: Waiting for close lock at 1731689965392Disabling compacts and flushes for region at 1731689965392Disabling writes for close at 1731689965393 (+1 ms)Writing region close event to WAL at 1731689965393Closed at 1731689965393 2024-11-15T16:59:25,394 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:59:25,394 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731689965394"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689965394"}]},"ts":"1731689965394"} 2024-11-15T16:59:25,395 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:59:25,396 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:59:25,396 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689965396"}]},"ts":"1731689965396"} 2024-11-15T16:59:25,397 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T16:59:25,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, ASSIGN}] 2024-11-15T16:59:25,398 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, ASSIGN 2024-11-15T16:59:25,399 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=false 2024-11-15T16:59:25,550 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:25,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, ASSIGN because future has completed 2024-11-15T16:59:25,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:59:25,707 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:25,707 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:59:25,707 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,707 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:59:25,707 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,707 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,709 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,710 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec8d5aa95a6ec0fca101912a0d79069 columnFamilyName A 2024-11-15T16:59:25,710 DEBUG [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:25,710 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(327): Store=fec8d5aa95a6ec0fca101912a0d79069/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:25,710 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,711 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec8d5aa95a6ec0fca101912a0d79069 columnFamilyName B 2024-11-15T16:59:25,711 DEBUG [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:25,712 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(327): Store=fec8d5aa95a6ec0fca101912a0d79069/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:25,712 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,713 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec8d5aa95a6ec0fca101912a0d79069 columnFamilyName C 2024-11-15T16:59:25,713 DEBUG [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:25,713 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(327): Store=fec8d5aa95a6ec0fca101912a0d79069/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:25,713 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,714 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,714 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,714 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,715 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,715 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:59:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T16:59:25,716 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,717 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:59:25,718 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened fec8d5aa95a6ec0fca101912a0d79069; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61633308, jitterRate=-0.0815921425819397}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:59:25,718 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:25,718 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for fec8d5aa95a6ec0fca101912a0d79069: Running coprocessor pre-open hook at 1731689965707Writing region info on filesystem at 1731689965708 (+1 ms)Initializing all the Stores at 1731689965708Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689965708Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689965708Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689965708Cleaning up temporary data from old regions at 1731689965715 (+7 ms)Running coprocessor post-open hooks at 1731689965718 (+3 ms)Region opened successfully at 1731689965718 2024-11-15T16:59:25,719 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., pid=97, masterSystemTime=1731689965704 2024-11-15T16:59:25,720 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:25,721 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:25,721 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:25,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:59:25,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-15T16:59:25,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 in 171 msec 2024-11-15T16:59:25,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T16:59:25,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, ASSIGN in 327 msec 2024-11-15T16:59:25,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:59:25,727 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689965727"}]},"ts":"1731689965727"} 2024-11-15T16:59:25,728 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T16:59:25,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:59:25,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-15T16:59:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T16:59:26,736 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:26,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:26,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:26,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:26,738 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a07eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:26,740 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e491331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:26,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:26,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:26,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:59:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:59:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62dc3ec5 2024-11-15T16:59:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:59:26,744 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:59:26,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T16:59:26,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:59:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:26,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742143_1319 (size=995) 2024-11-15T16:59:27,157 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T16:59:27,157 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T16:59:27,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T16:59:27,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, REOPEN/MOVE}] 2024-11-15T16:59:27,163 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, REOPEN/MOVE 2024-11-15T16:59:27,163 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, REOPEN/MOVE because future has completed 2024-11-15T16:59:27,165 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:59:27,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:59:27,317 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,317 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:59:27,317 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing fec8d5aa95a6ec0fca101912a0d79069, disabling compactions & flushes 2024-11-15T16:59:27,317 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,317 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,317 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. after waiting 0 ms 2024-11-15T16:59:27,317 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,321 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T16:59:27,321 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,321 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for fec8d5aa95a6ec0fca101912a0d79069: Waiting for close lock at 1731689967317Running coprocessor pre-close hooks at 1731689967317Disabling compacts and flushes for region at 1731689967317Disabling writes for close at 1731689967317Writing region close event to WAL at 1731689967318 (+1 ms)Running coprocessor post-close hooks at 1731689967321 (+3 ms)Closed at 1731689967321 2024-11-15T16:59:27,321 WARN [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: fec8d5aa95a6ec0fca101912a0d79069 to self. 2024-11-15T16:59:27,323 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,323 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=CLOSED 2024-11-15T16:59:27,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:59:27,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-15T16:59:27,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 in 160 msec 2024-11-15T16:59:27,327 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, REOPEN/MOVE; state=CLOSED, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=true 2024-11-15T16:59:27,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, REOPEN/MOVE because future has completed 2024-11-15T16:59:27,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:59:27,634 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:59:27,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:59:27,635 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,635 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,636 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,636 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec8d5aa95a6ec0fca101912a0d79069 columnFamilyName A 2024-11-15T16:59:27,637 DEBUG [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:27,638 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(327): Store=fec8d5aa95a6ec0fca101912a0d79069/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:27,638 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,638 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec8d5aa95a6ec0fca101912a0d79069 columnFamilyName B 2024-11-15T16:59:27,638 DEBUG [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:27,639 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(327): Store=fec8d5aa95a6ec0fca101912a0d79069/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:27,639 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,639 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec8d5aa95a6ec0fca101912a0d79069 columnFamilyName C 2024-11-15T16:59:27,639 DEBUG [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:27,639 INFO [StoreOpener-fec8d5aa95a6ec0fca101912a0d79069-1 {}] regionserver.HStore(327): Store=fec8d5aa95a6ec0fca101912a0d79069/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:27,640 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,640 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,641 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,641 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,642 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,642 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:59:27,643 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,643 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened fec8d5aa95a6ec0fca101912a0d79069; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68897696, jitterRate=0.02665567398071289}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:59:27,643 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,644 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for fec8d5aa95a6ec0fca101912a0d79069: Running coprocessor pre-open hook at 1731689967635Writing region info on filesystem at 1731689967635Initializing all the Stores at 1731689967635Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689967635Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689967635Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689967635Cleaning up temporary data from old regions at 1731689967642 (+7 ms)Running coprocessor post-open hooks at 1731689967643 (+1 ms)Region opened successfully at 1731689967644 (+1 ms) 2024-11-15T16:59:27,645 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., pid=102, masterSystemTime=1731689967631 2024-11-15T16:59:27,646 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,646 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,647 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=OPEN, openSeqNum=5, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:59:27,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-15T16:59:27,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 in 169 msec 2024-11-15T16:59:27,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T16:59:27,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, REOPEN/MOVE in 488 msec 2024-11-15T16:59:27,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-15T16:59:27,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-15T16:59:27,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-15T16:59:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-15T16:59:27,656 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:27,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cce248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,659 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb0426d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,661 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1932db06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6d67a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e044d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,665 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24c479f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,666 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f320ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c60e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,669 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5966f90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9300bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,672 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29471d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@83595b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,674 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258922c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,677 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351da4b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f0d003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,680 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3572d9ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac1ffed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:27,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:27,685 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e3600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:27,686 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@971da05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:27,687 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,687 DEBUG [Thread-1480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:27,687 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,688 DEBUG [Thread-1467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,688 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,689 DEBUG [Thread-1465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T16:59:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,689 DEBUG [Thread-1463 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,690 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,691 DEBUG [Thread-1469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,691 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:27,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,691 DEBUG [Thread-1471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,692 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T16:59:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T16:59:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,707 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:27,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:59:27,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111561a313145c1f4b329e4d8e9cd643b778_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689967703/Put/seqid=0 2024-11-15T16:59:27,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742144_1320 (size=12154) 2024-11-15T16:59:27,754 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:27,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111561a313145c1f4b329e4d8e9cd643b778_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111561a313145c1f4b329e4d8e9cd643b778_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:27,761 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/9002d20928ba4cfb8a155ccf38cdf94b, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:27,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/9002d20928ba4cfb8a155ccf38cdf94b is 175, key is test_row_0/A:col10/1731689967703/Put/seqid=0 2024-11-15T16:59:27,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690027756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690027757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742145_1321 (size=30955) 2024-11-15T16:59:27,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690027760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690027760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690027768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T16:59:27,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T16:59:27,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:27,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:27,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:27,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:27,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690027875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690027875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690027876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690027876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:27,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690027886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:27,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:27,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T16:59:28,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:28,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T16:59:28,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690028086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690028096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690028096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690028096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690028105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T16:59:28,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:28,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/9002d20928ba4cfb8a155ccf38cdf94b 2024-11-15T16:59:28,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/19549cb6c66b4b01903485eb76e5a27d is 50, key is test_row_0/B:col10/1731689967703/Put/seqid=0 2024-11-15T16:59:28,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742146_1322 (size=12001) 2024-11-15T16:59:28,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T16:59:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T16:59:28,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690028406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690028416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690028416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690028416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690028416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T16:59:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/19549cb6c66b4b01903485eb76e5a27d 2024-11-15T16:59:28,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T16:59:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:28,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/52df3cabc4104737a3a19d43edfe3688 is 50, key is test_row_0/C:col10/1731689967703/Put/seqid=0 2024-11-15T16:59:28,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742147_1323 (size=12001) 2024-11-15T16:59:28,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/52df3cabc4104737a3a19d43edfe3688 2024-11-15T16:59:28,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/9002d20928ba4cfb8a155ccf38cdf94b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/9002d20928ba4cfb8a155ccf38cdf94b 2024-11-15T16:59:28,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/9002d20928ba4cfb8a155ccf38cdf94b, entries=150, sequenceid=15, filesize=30.2 K 2024-11-15T16:59:28,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/19549cb6c66b4b01903485eb76e5a27d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/19549cb6c66b4b01903485eb76e5a27d 2024-11-15T16:59:28,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/19549cb6c66b4b01903485eb76e5a27d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T16:59:28,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/52df3cabc4104737a3a19d43edfe3688 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/52df3cabc4104737a3a19d43edfe3688 2024-11-15T16:59:28,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/52df3cabc4104737a3a19d43edfe3688, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T16:59:28,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fec8d5aa95a6ec0fca101912a0d79069 in 938ms, sequenceid=15, compaction requested=false 2024-11-15T16:59:28,645 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T16:59:28,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:28,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T16:59:28,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:28,767 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:59:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115138b449ee1b4433fb87435f98bce558d_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689967763/Put/seqid=0 2024-11-15T16:59:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742148_1324 (size=12154) 2024-11-15T16:59:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T16:59:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:28,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:28,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690028937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690028940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690028941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690028942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690028943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690029056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690029056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690029056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690029056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690029057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:29,182 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115138b449ee1b4433fb87435f98bce558d_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115138b449ee1b4433fb87435f98bce558d_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b59e259c86a14f0aaf6688ecdd66e5bf, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:29,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b59e259c86a14f0aaf6688ecdd66e5bf is 175, key is test_row_0/A:col10/1731689967763/Put/seqid=0 2024-11-15T16:59:29,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742149_1325 (size=30955) 2024-11-15T16:59:29,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690029265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690029266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690029266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690029266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690029266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,317 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T16:59:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690029575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690029576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690029576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690029576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,589 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b59e259c86a14f0aaf6688ecdd66e5bf 2024-11-15T16:59:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690029586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/323372814e5e4d588c7a7f87d810e912 is 50, key is test_row_0/B:col10/1731689967763/Put/seqid=0 2024-11-15T16:59:29,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742150_1326 (size=12001) 2024-11-15T16:59:29,614 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/323372814e5e4d588c7a7f87d810e912 2024-11-15T16:59:29,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/474672473dd948faa7494089fe1b8799 is 50, key is test_row_0/C:col10/1731689967763/Put/seqid=0 2024-11-15T16:59:29,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742151_1327 (size=12001) 2024-11-15T16:59:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T16:59:30,028 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/474672473dd948faa7494089fe1b8799 2024-11-15T16:59:30,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b59e259c86a14f0aaf6688ecdd66e5bf as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b59e259c86a14f0aaf6688ecdd66e5bf 2024-11-15T16:59:30,045 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b59e259c86a14f0aaf6688ecdd66e5bf, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T16:59:30,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/323372814e5e4d588c7a7f87d810e912 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/323372814e5e4d588c7a7f87d810e912 2024-11-15T16:59:30,050 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/323372814e5e4d588c7a7f87d810e912, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T16:59:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/474672473dd948faa7494089fe1b8799 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/474672473dd948faa7494089fe1b8799 2024-11-15T16:59:30,056 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/474672473dd948faa7494089fe1b8799, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T16:59:30,057 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fec8d5aa95a6ec0fca101912a0d79069 in 1290ms, sequenceid=40, compaction requested=false 2024-11-15T16:59:30,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:30,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:30,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T16:59:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T16:59:30,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T16:59:30,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-11-15T16:59:30,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3770 sec 2024-11-15T16:59:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:30,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T16:59:30,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d36829bd9cd3437ebbac03a3e7297663_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689970088/Put/seqid=0 2024-11-15T16:59:30,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742152_1328 (size=14594) 2024-11-15T16:59:30,115 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:30,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d36829bd9cd3437ebbac03a3e7297663_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d36829bd9cd3437ebbac03a3e7297663_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:30,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d6cb6bd9c619456097be8c8d494a68ae, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:30,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d6cb6bd9c619456097be8c8d494a68ae is 175, key is test_row_0/A:col10/1731689970088/Put/seqid=0 2024-11-15T16:59:30,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742153_1329 (size=39549) 2024-11-15T16:59:30,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d6cb6bd9c619456097be8c8d494a68ae 2024-11-15T16:59:30,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc is 50, key is test_row_0/B:col10/1731689970088/Put/seqid=0 2024-11-15T16:59:30,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742154_1330 (size=12001) 2024-11-15T16:59:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690030133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690030135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690030136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690030144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690030147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690030255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690030255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690030256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690030256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690030266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690030475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690030476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690030476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690030476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690030485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc 2024-11-15T16:59:30,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/e168d4e69c0c45c389da6b5ed29c7044 is 50, key is test_row_0/C:col10/1731689970088/Put/seqid=0 2024-11-15T16:59:30,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742155_1331 (size=12001) 2024-11-15T16:59:30,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690030785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690030796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690030796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690030796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690030796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:30,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/e168d4e69c0c45c389da6b5ed29c7044 2024-11-15T16:59:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d6cb6bd9c619456097be8c8d494a68ae as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d6cb6bd9c619456097be8c8d494a68ae 2024-11-15T16:59:30,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d6cb6bd9c619456097be8c8d494a68ae, entries=200, sequenceid=53, filesize=38.6 K 2024-11-15T16:59:30,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc 2024-11-15T16:59:30,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T16:59:30,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/e168d4e69c0c45c389da6b5ed29c7044 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e168d4e69c0c45c389da6b5ed29c7044 2024-11-15T16:59:30,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e168d4e69c0c45c389da6b5ed29c7044, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T16:59:30,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fec8d5aa95a6ec0fca101912a0d79069 in 893ms, sequenceid=53, compaction requested=true 2024-11-15T16:59:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:30,984 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:30,984 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:30,985 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:30,985 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:30,985 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/B is initiating minor compaction (all files) 2024-11-15T16:59:30,985 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/A is initiating minor compaction (all files) 2024-11-15T16:59:30,985 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/B in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:30,985 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/A in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:30,985 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/19549cb6c66b4b01903485eb76e5a27d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/323372814e5e4d588c7a7f87d810e912, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=35.2 K 2024-11-15T16:59:30,985 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/9002d20928ba4cfb8a155ccf38cdf94b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b59e259c86a14f0aaf6688ecdd66e5bf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d6cb6bd9c619456097be8c8d494a68ae] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=99.1 K 2024-11-15T16:59:30,985 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:30,985 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/9002d20928ba4cfb8a155ccf38cdf94b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b59e259c86a14f0aaf6688ecdd66e5bf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d6cb6bd9c619456097be8c8d494a68ae] 2024-11-15T16:59:30,985 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 19549cb6c66b4b01903485eb76e5a27d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731689967703 2024-11-15T16:59:30,986 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9002d20928ba4cfb8a155ccf38cdf94b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731689967703 2024-11-15T16:59:30,986 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 323372814e5e4d588c7a7f87d810e912, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731689967754 2024-11-15T16:59:30,986 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59e259c86a14f0aaf6688ecdd66e5bf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731689967754 2024-11-15T16:59:30,986 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 06bb4d1dfe9f4a5a99f008d2bbf9bfcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731689968939 2024-11-15T16:59:30,986 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6cb6bd9c619456097be8c8d494a68ae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731689968926 2024-11-15T16:59:30,993 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:30,993 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/f75816607a4d4d75aeb47d3aaf071b4d is 50, key is test_row_0/B:col10/1731689970088/Put/seqid=0 2024-11-15T16:59:30,995 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:30,997 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115478c039ab6ea448197180b596571b10b_fec8d5aa95a6ec0fca101912a0d79069 store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:30,998 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115478c039ab6ea448197180b596571b10b_fec8d5aa95a6ec0fca101912a0d79069, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:30,999 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115478c039ab6ea448197180b596571b10b_fec8d5aa95a6ec0fca101912a0d79069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:31,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742156_1332 (size=12104) 2024-11-15T16:59:31,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742157_1333 (size=4469) 2024-11-15T16:59:31,006 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#A#compaction#283 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:31,007 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/baa27d7d2f1141b886532352083a1788 is 175, key is test_row_0/A:col10/1731689970088/Put/seqid=0 2024-11-15T16:59:31,008 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/f75816607a4d4d75aeb47d3aaf071b4d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/f75816607a4d4d75aeb47d3aaf071b4d 2024-11-15T16:59:31,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742158_1334 (size=31058) 2024-11-15T16:59:31,014 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/B of fec8d5aa95a6ec0fca101912a0d79069 into f75816607a4d4d75aeb47d3aaf071b4d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:31,014 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:31,014 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/B, priority=13, startTime=1731689970984; duration=0sec 2024-11-15T16:59:31,014 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:31,014 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:B 2024-11-15T16:59:31,014 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:31,015 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:31,015 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/C is initiating minor compaction (all files) 2024-11-15T16:59:31,015 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/C in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:31,015 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/52df3cabc4104737a3a19d43edfe3688, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/474672473dd948faa7494089fe1b8799, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e168d4e69c0c45c389da6b5ed29c7044] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=35.2 K 2024-11-15T16:59:31,016 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 52df3cabc4104737a3a19d43edfe3688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731689967703 2024-11-15T16:59:31,016 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 474672473dd948faa7494089fe1b8799, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731689967754 2024-11-15T16:59:31,017 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e168d4e69c0c45c389da6b5ed29c7044, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731689968939 2024-11-15T16:59:31,023 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:31,023 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/1c7a754dcc794796bd9e6da94532e54f is 50, key is test_row_0/C:col10/1731689970088/Put/seqid=0 2024-11-15T16:59:31,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742159_1335 (size=12104) 2024-11-15T16:59:31,300 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:31,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T16:59:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:31,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bfa7d812289d4664bca585f040894899_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689970145/Put/seqid=0 2024-11-15T16:59:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742160_1336 (size=14594) 2024-11-15T16:59:31,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:31,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bfa7d812289d4664bca585f040894899_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfa7d812289d4664bca585f040894899_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:31,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690031320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,328 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/2bbc526c11634dc2b9f348f99fac59a7, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/2bbc526c11634dc2b9f348f99fac59a7 is 175, key is test_row_0/A:col10/1731689970145/Put/seqid=0 2024-11-15T16:59:31,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742161_1337 (size=39549) 2024-11-15T16:59:31,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/2bbc526c11634dc2b9f348f99fac59a7 2024-11-15T16:59:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690031321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690031323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690031328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690031328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/e038d9330a9e4b36bd356d4d69239d94 is 50, key is test_row_0/B:col10/1731689970145/Put/seqid=0 2024-11-15T16:59:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742162_1338 (size=12001) 2024-11-15T16:59:31,419 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/baa27d7d2f1141b886532352083a1788 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/baa27d7d2f1141b886532352083a1788 2024-11-15T16:59:31,424 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/A of fec8d5aa95a6ec0fca101912a0d79069 into baa27d7d2f1141b886532352083a1788(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:31,424 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:31,424 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/A, priority=13, startTime=1731689970984; duration=0sec 2024-11-15T16:59:31,425 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:31,425 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:A 2024-11-15T16:59:31,431 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/1c7a754dcc794796bd9e6da94532e54f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/1c7a754dcc794796bd9e6da94532e54f 2024-11-15T16:59:31,435 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/C of fec8d5aa95a6ec0fca101912a0d79069 into 1c7a754dcc794796bd9e6da94532e54f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:31,435 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:31,435 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/C, priority=13, startTime=1731689970984; duration=0sec 2024-11-15T16:59:31,435 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:31,435 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:C 2024-11-15T16:59:31,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690031435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690031446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690031446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690031446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690031446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690031656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690031656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690031656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690031656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690031656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/e038d9330a9e4b36bd356d4d69239d94 2024-11-15T16:59:31,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/5fb07e3ec6234d7886bebf5c7811ec3f is 50, key is test_row_0/C:col10/1731689970145/Put/seqid=0 2024-11-15T16:59:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742163_1339 (size=12001) 2024-11-15T16:59:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T16:59:31,847 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:31,847 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T16:59:31,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T16:59:31,850 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:31,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:31,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T16:59:31,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690031966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690031966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690031966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690031975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690031975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T16:59:32,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:32,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:32,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:32,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:32,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:32,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T16:59:32,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:32,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:32,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:32,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:32,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T16:59:32,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/5fb07e3ec6234d7886bebf5c7811ec3f 2024-11-15T16:59:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/2bbc526c11634dc2b9f348f99fac59a7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2bbc526c11634dc2b9f348f99fac59a7 2024-11-15T16:59:32,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2bbc526c11634dc2b9f348f99fac59a7, entries=200, sequenceid=78, filesize=38.6 K 2024-11-15T16:59:32,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/e038d9330a9e4b36bd356d4d69239d94 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e038d9330a9e4b36bd356d4d69239d94 2024-11-15T16:59:32,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e038d9330a9e4b36bd356d4d69239d94, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T16:59:32,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/5fb07e3ec6234d7886bebf5c7811ec3f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/5fb07e3ec6234d7886bebf5c7811ec3f 2024-11-15T16:59:32,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/5fb07e3ec6234d7886bebf5c7811ec3f, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T16:59:32,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for fec8d5aa95a6ec0fca101912a0d79069 in 892ms, sequenceid=78, compaction requested=false 2024-11-15T16:59:32,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:32,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T16:59:32,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:32,312 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T16:59:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111548355313d24a4cfa93586887cccca16a_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689971321/Put/seqid=0 2024-11-15T16:59:32,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742164_1340 (size=12154) 2024-11-15T16:59:32,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:32,328 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111548355313d24a4cfa93586887cccca16a_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111548355313d24a4cfa93586887cccca16a_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b7cd2dcf834a4204843e319a37d627fe, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b7cd2dcf834a4204843e319a37d627fe is 175, key is test_row_0/A:col10/1731689971321/Put/seqid=0 2024-11-15T16:59:32,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742165_1341 (size=30955) 2024-11-15T16:59:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T16:59:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:32,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:32,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690032530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690032531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690032532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690032533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690032534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690032646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690032646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690032646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690032656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690032656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,733 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b7cd2dcf834a4204843e319a37d627fe 2024-11-15T16:59:32,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/b6aed81760c146ec965b5a427f74fc64 is 50, key is test_row_0/B:col10/1731689971321/Put/seqid=0 2024-11-15T16:59:32,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742166_1342 (size=12001) 2024-11-15T16:59:32,750 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/b6aed81760c146ec965b5a427f74fc64 2024-11-15T16:59:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/6bf03b6f6cfe4f109c1c0dc287c7df6e is 50, key is test_row_0/C:col10/1731689971321/Put/seqid=0 2024-11-15T16:59:32,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742167_1343 (size=12001) 2024-11-15T16:59:32,763 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/6bf03b6f6cfe4f109c1c0dc287c7df6e 2024-11-15T16:59:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/b7cd2dcf834a4204843e319a37d627fe as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b7cd2dcf834a4204843e319a37d627fe 2024-11-15T16:59:32,771 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b7cd2dcf834a4204843e319a37d627fe, entries=150, sequenceid=93, filesize=30.2 K 2024-11-15T16:59:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/b6aed81760c146ec965b5a427f74fc64 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/b6aed81760c146ec965b5a427f74fc64 2024-11-15T16:59:32,775 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/b6aed81760c146ec965b5a427f74fc64, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T16:59:32,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/6bf03b6f6cfe4f109c1c0dc287c7df6e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/6bf03b6f6cfe4f109c1c0dc287c7df6e 2024-11-15T16:59:32,780 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/6bf03b6f6cfe4f109c1c0dc287c7df6e, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T16:59:32,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fec8d5aa95a6ec0fca101912a0d79069 in 470ms, sequenceid=93, compaction requested=true 2024-11-15T16:59:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T16:59:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T16:59:32,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T16:59:32,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 931 msec 2024-11-15T16:59:32,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 938 msec 2024-11-15T16:59:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:32,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:32,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T16:59:32,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158c0d3e411a7e4f8388492cc22a537426_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689972863/Put/seqid=0 2024-11-15T16:59:32,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742168_1344 (size=17034) 2024-11-15T16:59:32,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690032874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690032876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690032877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690032878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690032883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T16:59:32,986 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:32,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-15T16:59:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T16:59:32,988 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:32,989 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:32,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:32,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690032985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690032996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690032996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690032996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690032996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T16:59:33,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:33,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:33,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690033196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690033216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690033216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690033216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690033216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:33,282 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158c0d3e411a7e4f8388492cc22a537426_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c0d3e411a7e4f8388492cc22a537426_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:33,283 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0db67e22bfb64fe88dc34a840c13b4a3, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:33,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0db67e22bfb64fe88dc34a840c13b4a3 is 175, key is test_row_0/A:col10/1731689972863/Put/seqid=0 2024-11-15T16:59:33,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742169_1345 (size=48139) 2024-11-15T16:59:33,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:33,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:33,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T16:59:33,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690033505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690033526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690033536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690033536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690033536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:33,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:33,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:33,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T16:59:33,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0db67e22bfb64fe88dc34a840c13b4a3 2024-11-15T16:59:33,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/196c983bf6044ac298efa63eb0c8f29d is 50, key is test_row_0/B:col10/1731689972863/Put/seqid=0 2024-11-15T16:59:33,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742170_1346 (size=12001) 2024-11-15T16:59:33,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:33,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:33,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690034016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690034035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690034045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690034045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690034045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:34,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/196c983bf6044ac298efa63eb0c8f29d 2024-11-15T16:59:34,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/0e4a3d4fa38b459b96f94576061b34ec is 50, key is test_row_0/C:col10/1731689972863/Put/seqid=0 2024-11-15T16:59:34,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742171_1347 (size=12001) 2024-11-15T16:59:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T16:59:34,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/0e4a3d4fa38b459b96f94576061b34ec 2024-11-15T16:59:34,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0db67e22bfb64fe88dc34a840c13b4a3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0db67e22bfb64fe88dc34a840c13b4a3 2024-11-15T16:59:34,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0db67e22bfb64fe88dc34a840c13b4a3, entries=250, sequenceid=119, filesize=47.0 K 2024-11-15T16:59:34,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/196c983bf6044ac298efa63eb0c8f29d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/196c983bf6044ac298efa63eb0c8f29d 2024-11-15T16:59:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:34,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/196c983bf6044ac298efa63eb0c8f29d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-15T16:59:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/0e4a3d4fa38b459b96f94576061b34ec as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0e4a3d4fa38b459b96f94576061b34ec 2024-11-15T16:59:34,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0e4a3d4fa38b459b96f94576061b34ec, entries=150, sequenceid=119, filesize=11.7 K 2024-11-15T16:59:34,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fec8d5aa95a6ec0fca101912a0d79069 in 1672ms, sequenceid=119, compaction requested=true 2024-11-15T16:59:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:34,536 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:34,536 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:34,537 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:34,537 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/A is initiating minor compaction (all files) 2024-11-15T16:59:34,537 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/A in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,537 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/baa27d7d2f1141b886532352083a1788, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2bbc526c11634dc2b9f348f99fac59a7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b7cd2dcf834a4204843e319a37d627fe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0db67e22bfb64fe88dc34a840c13b4a3] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=146.2 K 2024-11-15T16:59:34,537 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:34,537 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,537 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/baa27d7d2f1141b886532352083a1788, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2bbc526c11634dc2b9f348f99fac59a7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b7cd2dcf834a4204843e319a37d627fe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0db67e22bfb64fe88dc34a840c13b4a3] 2024-11-15T16:59:34,538 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/B is initiating minor compaction (all files) 2024-11-15T16:59:34,538 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/B in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,538 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/f75816607a4d4d75aeb47d3aaf071b4d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e038d9330a9e4b36bd356d4d69239d94, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/b6aed81760c146ec965b5a427f74fc64, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/196c983bf6044ac298efa63eb0c8f29d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=47.0 K 2024-11-15T16:59:34,538 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa27d7d2f1141b886532352083a1788, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731689968939 2024-11-15T16:59:34,538 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f75816607a4d4d75aeb47d3aaf071b4d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731689968939 2024-11-15T16:59:34,538 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bbc526c11634dc2b9f348f99fac59a7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731689970106 2024-11-15T16:59:34,539 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e038d9330a9e4b36bd356d4d69239d94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731689970106 2024-11-15T16:59:34,539 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7cd2dcf834a4204843e319a37d627fe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731689971321 2024-11-15T16:59:34,539 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0db67e22bfb64fe88dc34a840c13b4a3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731689972519 2024-11-15T16:59:34,539 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting b6aed81760c146ec965b5a427f74fc64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731689971321 2024-11-15T16:59:34,540 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 196c983bf6044ac298efa63eb0c8f29d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731689972861 2024-11-15T16:59:34,559 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:34,559 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/e81bff7ea38d4dc482b06ea4eb534920 is 50, key is test_row_0/B:col10/1731689972863/Put/seqid=0 2024-11-15T16:59:34,561 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:34,570 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111580ff8500682d45e2af83b55e2f73f54e_fec8d5aa95a6ec0fca101912a0d79069 store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:34,573 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111580ff8500682d45e2af83b55e2f73f54e_fec8d5aa95a6ec0fca101912a0d79069, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:34,573 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111580ff8500682d45e2af83b55e2f73f54e_fec8d5aa95a6ec0fca101912a0d79069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:34,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742172_1348 (size=12241) 2024-11-15T16:59:34,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742173_1349 (size=4469) 2024-11-15T16:59:34,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T16:59:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,677 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T16:59:34,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157259d464417e4c3ebf22f48c156f54e3_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689972882/Put/seqid=0 2024-11-15T16:59:34,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742174_1350 (size=12154) 2024-11-15T16:59:34,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:34,695 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157259d464417e4c3ebf22f48c156f54e3_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157259d464417e4c3ebf22f48c156f54e3_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e33a19c4e3e4476f893db358827cdbf0, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e33a19c4e3e4476f893db358827cdbf0 is 175, key is test_row_0/A:col10/1731689972882/Put/seqid=0 2024-11-15T16:59:34,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742175_1351 (size=30955) 2024-11-15T16:59:34,977 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/e81bff7ea38d4dc482b06ea4eb534920 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e81bff7ea38d4dc482b06ea4eb534920 2024-11-15T16:59:34,982 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/B of fec8d5aa95a6ec0fca101912a0d79069 into e81bff7ea38d4dc482b06ea4eb534920(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:34,982 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:34,982 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/B, priority=12, startTime=1731689974536; duration=0sec 2024-11-15T16:59:34,982 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:34,982 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:B 2024-11-15T16:59:34,982 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:34,984 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:34,984 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/C is initiating minor compaction (all files) 2024-11-15T16:59:34,984 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/C in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:34,984 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/1c7a754dcc794796bd9e6da94532e54f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/5fb07e3ec6234d7886bebf5c7811ec3f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/6bf03b6f6cfe4f109c1c0dc287c7df6e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0e4a3d4fa38b459b96f94576061b34ec] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=47.0 K 2024-11-15T16:59:34,985 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7a754dcc794796bd9e6da94532e54f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731689968939 2024-11-15T16:59:34,985 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb07e3ec6234d7886bebf5c7811ec3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731689970106 2024-11-15T16:59:34,985 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf03b6f6cfe4f109c1c0dc287c7df6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731689971321 2024-11-15T16:59:34,986 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e4a3d4fa38b459b96f94576061b34ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731689972861 2024-11-15T16:59:34,987 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:34,987 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/aa9b2d3cb17a43c28407ed4b6ff533f6 is 175, key is test_row_0/A:col10/1731689972863/Put/seqid=0 2024-11-15T16:59:35,003 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:35,004 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/212e073fe955481789e9d079bd9032b5 is 50, key is test_row_0/C:col10/1731689972863/Put/seqid=0 2024-11-15T16:59:35,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742176_1352 (size=31195) 2024-11-15T16:59:35,016 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/aa9b2d3cb17a43c28407ed4b6ff533f6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/aa9b2d3cb17a43c28407ed4b6ff533f6 2024-11-15T16:59:35,025 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/A of fec8d5aa95a6ec0fca101912a0d79069 into aa9b2d3cb17a43c28407ed4b6ff533f6(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:35,025 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:35,025 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/A, priority=12, startTime=1731689974536; duration=0sec 2024-11-15T16:59:35,025 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:35,025 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:A 2024-11-15T16:59:35,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742177_1353 (size=12241) 2024-11-15T16:59:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:35,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:35,039 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/212e073fe955481789e9d079bd9032b5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/212e073fe955481789e9d079bd9032b5 2024-11-15T16:59:35,047 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/C of fec8d5aa95a6ec0fca101912a0d79069 into 212e073fe955481789e9d079bd9032b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:35,047 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:35,047 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/C, priority=12, startTime=1731689974536; duration=0sec 2024-11-15T16:59:35,047 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:35,047 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:C 2024-11-15T16:59:35,105 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e33a19c4e3e4476f893db358827cdbf0 2024-11-15T16:59:35,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690035108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690035110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690035113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/9efe4696d7ee482a8e2764c1384a6017 is 50, key is test_row_0/B:col10/1731689972882/Put/seqid=0 2024-11-15T16:59:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690035116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690035118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T16:59:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742178_1354 (size=12001) 2024-11-15T16:59:35,142 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/9efe4696d7ee482a8e2764c1384a6017 2024-11-15T16:59:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/62e8af28df8e461c8f1c930e5de98ac6 is 50, key is test_row_0/C:col10/1731689972882/Put/seqid=0 2024-11-15T16:59:35,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742179_1355 (size=12001) 2024-11-15T16:59:35,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690035225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690035225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690035225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690035236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690035236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690035435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690035435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690035435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690035456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690035456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,555 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/62e8af28df8e461c8f1c930e5de98ac6 2024-11-15T16:59:35,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e33a19c4e3e4476f893db358827cdbf0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e33a19c4e3e4476f893db358827cdbf0 2024-11-15T16:59:35,564 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e33a19c4e3e4476f893db358827cdbf0, entries=150, sequenceid=129, filesize=30.2 K 2024-11-15T16:59:35,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/9efe4696d7ee482a8e2764c1384a6017 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/9efe4696d7ee482a8e2764c1384a6017 2024-11-15T16:59:35,569 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/9efe4696d7ee482a8e2764c1384a6017, entries=150, sequenceid=129, filesize=11.7 K 2024-11-15T16:59:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/62e8af28df8e461c8f1c930e5de98ac6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/62e8af28df8e461c8f1c930e5de98ac6 2024-11-15T16:59:35,574 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/62e8af28df8e461c8f1c930e5de98ac6, entries=150, sequenceid=129, filesize=11.7 K 2024-11-15T16:59:35,575 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for fec8d5aa95a6ec0fca101912a0d79069 in 898ms, sequenceid=129, compaction requested=false 2024-11-15T16:59:35,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:35,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:35,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-15T16:59:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-15T16:59:35,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T16:59:35,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5870 sec 2024-11-15T16:59:35,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.5920 sec 2024-11-15T16:59:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:35,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:35,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T16:59:35,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a9cc948ae7da4b6ba94370cbbf82dce8_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689975750/Put/seqid=0 2024-11-15T16:59:35,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742180_1356 (size=14794) 2024-11-15T16:59:35,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690035765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690035766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690035767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690035768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690035776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690035886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690035886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:35,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690035886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690036096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690036096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690036096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:36,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a9cc948ae7da4b6ba94370cbbf82dce8_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a9cc948ae7da4b6ba94370cbbf82dce8_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:36,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/19d760ae2e7e4f8fb6f15a465194b780, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:36,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/19d760ae2e7e4f8fb6f15a465194b780 is 175, key is test_row_0/A:col10/1731689975750/Put/seqid=0 2024-11-15T16:59:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742181_1357 (size=39749) 2024-11-15T16:59:36,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/19d760ae2e7e4f8fb6f15a465194b780 2024-11-15T16:59:36,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/77f02daa19764f7ebfba1bc9075a9d8c is 50, key is test_row_0/B:col10/1731689975750/Put/seqid=0 2024-11-15T16:59:36,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742182_1358 (size=12151) 2024-11-15T16:59:36,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690036285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690036285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690036405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690036405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690036406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/77f02daa19764f7ebfba1bc9075a9d8c 2024-11-15T16:59:36,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/48d5cabba75541d9a80c324a2508ee98 is 50, key is test_row_0/C:col10/1731689975750/Put/seqid=0 2024-11-15T16:59:36,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742183_1359 (size=12151) 2024-11-15T16:59:36,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/48d5cabba75541d9a80c324a2508ee98 2024-11-15T16:59:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/19d760ae2e7e4f8fb6f15a465194b780 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/19d760ae2e7e4f8fb6f15a465194b780 2024-11-15T16:59:36,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/19d760ae2e7e4f8fb6f15a465194b780, entries=200, sequenceid=159, filesize=38.8 K 2024-11-15T16:59:36,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/77f02daa19764f7ebfba1bc9075a9d8c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/77f02daa19764f7ebfba1bc9075a9d8c 2024-11-15T16:59:36,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/77f02daa19764f7ebfba1bc9075a9d8c, entries=150, sequenceid=159, filesize=11.9 K 2024-11-15T16:59:36,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/48d5cabba75541d9a80c324a2508ee98 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/48d5cabba75541d9a80c324a2508ee98 2024-11-15T16:59:36,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/48d5cabba75541d9a80c324a2508ee98, entries=150, sequenceid=159, filesize=11.9 K 2024-11-15T16:59:36,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for fec8d5aa95a6ec0fca101912a0d79069 in 871ms, sequenceid=159, compaction requested=true 2024-11-15T16:59:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:36,622 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:36,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:36,622 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:36,624 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:36,624 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/A is initiating minor compaction (all files) 2024-11-15T16:59:36,624 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:36,624 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/A in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:36,624 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/B is initiating minor compaction (all files) 2024-11-15T16:59:36,624 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/aa9b2d3cb17a43c28407ed4b6ff533f6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e33a19c4e3e4476f893db358827cdbf0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/19d760ae2e7e4f8fb6f15a465194b780] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=99.5 K 2024-11-15T16:59:36,624 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/B in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:36,624 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:36,624 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/aa9b2d3cb17a43c28407ed4b6ff533f6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e33a19c4e3e4476f893db358827cdbf0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/19d760ae2e7e4f8fb6f15a465194b780] 2024-11-15T16:59:36,624 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e81bff7ea38d4dc482b06ea4eb534920, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/9efe4696d7ee482a8e2764c1384a6017, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/77f02daa19764f7ebfba1bc9075a9d8c] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=35.5 K 2024-11-15T16:59:36,625 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9b2d3cb17a43c28407ed4b6ff533f6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731689972861 2024-11-15T16:59:36,625 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e81bff7ea38d4dc482b06ea4eb534920, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731689972861 2024-11-15T16:59:36,625 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33a19c4e3e4476f893db358827cdbf0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731689972875 2024-11-15T16:59:36,625 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 9efe4696d7ee482a8e2764c1384a6017, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731689972875 2024-11-15T16:59:36,626 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d760ae2e7e4f8fb6f15a465194b780, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731689975111 2024-11-15T16:59:36,626 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f02daa19764f7ebfba1bc9075a9d8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731689975111 2024-11-15T16:59:36,633 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:36,633 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/85c4f15ec7c34b7aba90d2d1ae928c7c is 50, key is test_row_0/B:col10/1731689975750/Put/seqid=0 2024-11-15T16:59:36,635 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:36,638 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e69b608da8ae4c8096e2d957e95ec342_fec8d5aa95a6ec0fca101912a0d79069 store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:36,640 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e69b608da8ae4c8096e2d957e95ec342_fec8d5aa95a6ec0fca101912a0d79069, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:36,640 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e69b608da8ae4c8096e2d957e95ec342_fec8d5aa95a6ec0fca101912a0d79069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:36,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742184_1360 (size=12493) 2024-11-15T16:59:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742185_1361 (size=4469) 2024-11-15T16:59:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:36,923 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:36,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:59:36,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150b96faf4db3c41d28db9ed33cb8e8ede_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689976922/Put/seqid=0 2024-11-15T16:59:36,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742186_1362 (size=12304) 2024-11-15T16:59:36,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690036981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690036982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690036984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,046 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/85c4f15ec7c34b7aba90d2d1ae928c7c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/85c4f15ec7c34b7aba90d2d1ae928c7c 2024-11-15T16:59:37,050 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/B of fec8d5aa95a6ec0fca101912a0d79069 into 85c4f15ec7c34b7aba90d2d1ae928c7c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:37,050 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:37,051 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/B, priority=13, startTime=1731689976622; duration=0sec 2024-11-15T16:59:37,051 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:37,051 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:B 2024-11-15T16:59:37,051 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:37,052 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:37,052 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:37,053 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/C is initiating minor compaction (all files) 2024-11-15T16:59:37,053 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/C in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,053 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/212e073fe955481789e9d079bd9032b5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/62e8af28df8e461c8f1c930e5de98ac6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/48d5cabba75541d9a80c324a2508ee98] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=35.5 K 2024-11-15T16:59:37,053 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/2b604dd0a3484e9196268cdd5c10c365 is 175, key is test_row_0/A:col10/1731689975750/Put/seqid=0 2024-11-15T16:59:37,053 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 212e073fe955481789e9d079bd9032b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731689972861 2024-11-15T16:59:37,053 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e8af28df8e461c8f1c930e5de98ac6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731689972875 2024-11-15T16:59:37,054 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d5cabba75541d9a80c324a2508ee98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731689975111 2024-11-15T16:59:37,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742187_1363 (size=31447) 2024-11-15T16:59:37,061 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:37,061 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/2b604dd0a3484e9196268cdd5c10c365 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2b604dd0a3484e9196268cdd5c10c365 2024-11-15T16:59:37,061 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/813d4504dc5c4f2e975d1ce131b1dc2e is 50, key is test_row_0/C:col10/1731689975750/Put/seqid=0 2024-11-15T16:59:37,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742188_1364 (size=12493) 2024-11-15T16:59:37,067 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/A of fec8d5aa95a6ec0fca101912a0d79069 into 2b604dd0a3484e9196268cdd5c10c365(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:37,067 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:37,067 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/A, priority=13, startTime=1731689976622; duration=0sec 2024-11-15T16:59:37,067 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:37,067 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:A 2024-11-15T16:59:37,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690037096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690037096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690037096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T16:59:37,156 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:37,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-15T16:59:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T16:59:37,159 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:37,159 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:37,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T16:59:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690037295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690037295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,299 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in fec8d5aa95a6ec0fca101912a0d79069 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,299 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in fec8d5aa95a6ec0fca101912a0d79069 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690037316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690037316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690037316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:37,339 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150b96faf4db3c41d28db9ed33cb8e8ede_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150b96faf4db3c41d28db9ed33cb8e8ede_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:37,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/7d926eda42aa4e95909124197748cfd0, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:37,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/7d926eda42aa4e95909124197748cfd0 is 175, key is test_row_0/A:col10/1731689976922/Put/seqid=0 2024-11-15T16:59:37,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742189_1365 (size=31105) 2024-11-15T16:59:37,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,470 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/813d4504dc5c4f2e975d1ce131b1dc2e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/813d4504dc5c4f2e975d1ce131b1dc2e 2024-11-15T16:59:37,475 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/C of fec8d5aa95a6ec0fca101912a0d79069 into 813d4504dc5c4f2e975d1ce131b1dc2e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:37,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:37,475 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/C, priority=13, startTime=1731689976622; duration=0sec 2024-11-15T16:59:37,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:37,475 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:C 2024-11-15T16:59:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T16:59:37,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690037625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690037626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690037626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:37,745 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/7d926eda42aa4e95909124197748cfd0 2024-11-15T16:59:37,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/8a9a49d61c3347e8a311787e36db8b62 is 50, key is test_row_0/B:col10/1731689976922/Put/seqid=0 2024-11-15T16:59:37,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742190_1366 (size=12151) 2024-11-15T16:59:37,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T16:59:37,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:37,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690038146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690038146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:38,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690038146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:38,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/8a9a49d61c3347e8a311787e36db8b62 2024-11-15T16:59:38,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/4e1d8e4d01fa47bbb17471acf3cf0665 is 50, key is test_row_0/C:col10/1731689976922/Put/seqid=0 2024-11-15T16:59:38,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742191_1367 (size=12151) 2024-11-15T16:59:38,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T16:59:38,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:38,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/4e1d8e4d01fa47bbb17471acf3cf0665 2024-11-15T16:59:38,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/7d926eda42aa4e95909124197748cfd0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/7d926eda42aa4e95909124197748cfd0 2024-11-15T16:59:38,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/7d926eda42aa4e95909124197748cfd0, entries=150, sequenceid=170, filesize=30.4 K 2024-11-15T16:59:38,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/8a9a49d61c3347e8a311787e36db8b62 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8a9a49d61c3347e8a311787e36db8b62 2024-11-15T16:59:38,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8a9a49d61c3347e8a311787e36db8b62, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T16:59:38,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/4e1d8e4d01fa47bbb17471acf3cf0665 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/4e1d8e4d01fa47bbb17471acf3cf0665 2024-11-15T16:59:38,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/4e1d8e4d01fa47bbb17471acf3cf0665, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T16:59:38,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fec8d5aa95a6ec0fca101912a0d79069 in 1662ms, sequenceid=170, compaction requested=false 2024-11-15T16:59:38,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:38,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T16:59:38,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:38,690 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:59:38,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e969f5e01189450db22b6416e1b17459_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689976981/Put/seqid=0 2024-11-15T16:59:38,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742192_1368 (size=12304) 2024-11-15T16:59:38,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:38,721 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e969f5e01189450db22b6416e1b17459_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e969f5e01189450db22b6416e1b17459_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:38,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/76616264647d430da232be5cc1a47afc, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:38,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/76616264647d430da232be5cc1a47afc is 175, key is test_row_0/A:col10/1731689976981/Put/seqid=0 2024-11-15T16:59:38,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742193_1369 (size=31105) 2024-11-15T16:59:39,128 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/76616264647d430da232be5cc1a47afc 2024-11-15T16:59:39,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/404d8ccf35434f9c98543646e8a7c04a is 50, key is test_row_0/B:col10/1731689976981/Put/seqid=0 2024-11-15T16:59:39,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742194_1370 (size=12151) 2024-11-15T16:59:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:39,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:39,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690039176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690039177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690039181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690039285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690039296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690039296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T16:59:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690039305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,312 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in fec8d5aa95a6ec0fca101912a0d79069 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690039326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,333 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in fec8d5aa95a6ec0fca101912a0d79069 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690039496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690039505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690039505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,544 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/404d8ccf35434f9c98543646e8a7c04a 2024-11-15T16:59:39,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/2beea1f69d694f329de0748c7fb67ba3 is 50, key is test_row_0/C:col10/1731689976981/Put/seqid=0 2024-11-15T16:59:39,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742195_1371 (size=12151) 2024-11-15T16:59:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690039806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690039816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690039816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:39,956 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/2beea1f69d694f329de0748c7fb67ba3 2024-11-15T16:59:39,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/76616264647d430da232be5cc1a47afc as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76616264647d430da232be5cc1a47afc 2024-11-15T16:59:39,964 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76616264647d430da232be5cc1a47afc, entries=150, sequenceid=198, filesize=30.4 K 2024-11-15T16:59:39,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/404d8ccf35434f9c98543646e8a7c04a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/404d8ccf35434f9c98543646e8a7c04a 2024-11-15T16:59:39,969 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/404d8ccf35434f9c98543646e8a7c04a, entries=150, sequenceid=198, filesize=11.9 K 2024-11-15T16:59:39,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/2beea1f69d694f329de0748c7fb67ba3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/2beea1f69d694f329de0748c7fb67ba3 2024-11-15T16:59:39,974 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/2beea1f69d694f329de0748c7fb67ba3, entries=150, sequenceid=198, filesize=11.9 K 2024-11-15T16:59:39,975 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fec8d5aa95a6ec0fca101912a0d79069 in 1285ms, sequenceid=198, compaction requested=true 2024-11-15T16:59:39,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:39,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:39,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-15T16:59:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-15T16:59:39,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T16:59:39,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8170 sec 2024-11-15T16:59:39,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.8220 sec 2024-11-15T16:59:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:40,321 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:40,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T16:59:40,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e89f0581914e46bdad73a5098a8ff88e_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689980320/Put/seqid=0 2024-11-15T16:59:40,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742196_1372 (size=14794) 2024-11-15T16:59:40,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690040382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690040385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690040391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690040495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690040506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690040506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690040705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690040716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690040716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:40,737 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:40,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e89f0581914e46bdad73a5098a8ff88e_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e89f0581914e46bdad73a5098a8ff88e_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:40,741 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/1e108181ce8c430db087c0e2c082d986, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:40,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/1e108181ce8c430db087c0e2c082d986 is 175, key is test_row_0/A:col10/1731689980320/Put/seqid=0 2024-11-15T16:59:40,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742197_1373 (size=39749) 2024-11-15T16:59:40,745 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/1e108181ce8c430db087c0e2c082d986 2024-11-15T16:59:40,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/1757ec1013034fef8578be862d29d173 is 50, key is test_row_0/B:col10/1731689980320/Put/seqid=0 2024-11-15T16:59:40,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742198_1374 (size=12151) 2024-11-15T16:59:40,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/1757ec1013034fef8578be862d29d173 2024-11-15T16:59:40,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/32415ca0067a4124911b4f3da38fd389 is 50, key is test_row_0/C:col10/1731689980320/Put/seqid=0 2024-11-15T16:59:40,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742199_1375 (size=12151) 2024-11-15T16:59:40,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/32415ca0067a4124911b4f3da38fd389 2024-11-15T16:59:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/1e108181ce8c430db087c0e2c082d986 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/1e108181ce8c430db087c0e2c082d986 2024-11-15T16:59:40,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/1e108181ce8c430db087c0e2c082d986, entries=200, sequenceid=210, filesize=38.8 K 2024-11-15T16:59:40,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/1757ec1013034fef8578be862d29d173 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1757ec1013034fef8578be862d29d173 2024-11-15T16:59:40,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1757ec1013034fef8578be862d29d173, entries=150, sequenceid=210, filesize=11.9 K 2024-11-15T16:59:40,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/32415ca0067a4124911b4f3da38fd389 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/32415ca0067a4124911b4f3da38fd389 2024-11-15T16:59:40,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/32415ca0067a4124911b4f3da38fd389, entries=150, sequenceid=210, filesize=11.9 K 2024-11-15T16:59:40,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fec8d5aa95a6ec0fca101912a0d79069 in 466ms, sequenceid=210, compaction requested=true 2024-11-15T16:59:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:40,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:40,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:40,788 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:40,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:40,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:40,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:40,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:40,788 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/A is initiating minor compaction (all files) 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/B is initiating minor compaction (all files) 2024-11-15T16:59:40,789 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/A in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:40,789 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/B in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:40,789 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2b604dd0a3484e9196268cdd5c10c365, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/7d926eda42aa4e95909124197748cfd0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76616264647d430da232be5cc1a47afc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/1e108181ce8c430db087c0e2c082d986] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=130.3 K 2024-11-15T16:59:40,789 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2b604dd0a3484e9196268cdd5c10c365, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/7d926eda42aa4e95909124197748cfd0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76616264647d430da232be5cc1a47afc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/1e108181ce8c430db087c0e2c082d986] 2024-11-15T16:59:40,789 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/85c4f15ec7c34b7aba90d2d1ae928c7c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8a9a49d61c3347e8a311787e36db8b62, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/404d8ccf35434f9c98543646e8a7c04a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1757ec1013034fef8578be862d29d173] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=47.8 K 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 85c4f15ec7c34b7aba90d2d1ae928c7c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731689975111 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b604dd0a3484e9196268cdd5c10c365, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731689975111 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d926eda42aa4e95909124197748cfd0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731689975763 2024-11-15T16:59:40,789 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9a49d61c3347e8a311787e36db8b62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731689975763 2024-11-15T16:59:40,790 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76616264647d430da232be5cc1a47afc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731689976971 2024-11-15T16:59:40,790 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 404d8ccf35434f9c98543646e8a7c04a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731689976971 2024-11-15T16:59:40,790 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e108181ce8c430db087c0e2c082d986, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731689979169 2024-11-15T16:59:40,790 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1757ec1013034fef8578be862d29d173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731689979169 2024-11-15T16:59:40,799 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:40,799 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/ae616b5871124b7ea9087f9e6fceece4 is 50, key is test_row_0/B:col10/1731689980320/Put/seqid=0 2024-11-15T16:59:40,803 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:40,809 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111577732437ace4430d9a5e32d7a59085a0_fec8d5aa95a6ec0fca101912a0d79069 store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:40,811 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111577732437ace4430d9a5e32d7a59085a0_fec8d5aa95a6ec0fca101912a0d79069, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:40,811 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111577732437ace4430d9a5e32d7a59085a0_fec8d5aa95a6ec0fca101912a0d79069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:40,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742200_1376 (size=12629) 2024-11-15T16:59:40,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742201_1377 (size=4469) 2024-11-15T16:59:40,824 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#A#compaction#316 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:40,824 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/76fd442fe5964cf7b7f974036b23bd51 is 175, key is test_row_0/A:col10/1731689980320/Put/seqid=0 2024-11-15T16:59:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742202_1378 (size=31583) 2024-11-15T16:59:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:41,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:41,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:59:41,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155d3aeb3097324874b5bc549859aa3ad0_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689980384/Put/seqid=0 2024-11-15T16:59:41,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742203_1379 (size=14794) 2024-11-15T16:59:41,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:41,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155d3aeb3097324874b5bc549859aa3ad0_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155d3aeb3097324874b5bc549859aa3ad0_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:41,046 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/a00ae1b8c85b4a49b24894aa7bbf76fa, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:41,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/a00ae1b8c85b4a49b24894aa7bbf76fa is 175, key is test_row_0/A:col10/1731689980384/Put/seqid=0 2024-11-15T16:59:41,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742204_1380 (size=39749) 2024-11-15T16:59:41,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690041054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690041058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690041066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690041176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690041176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690041186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,222 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/ae616b5871124b7ea9087f9e6fceece4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/ae616b5871124b7ea9087f9e6fceece4 2024-11-15T16:59:41,226 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/B of fec8d5aa95a6ec0fca101912a0d79069 into ae616b5871124b7ea9087f9e6fceece4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:41,226 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:41,226 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/B, priority=12, startTime=1731689980788; duration=0sec 2024-11-15T16:59:41,226 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:41,226 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:B 2024-11-15T16:59:41,227 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:41,228 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:41,228 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/C is initiating minor compaction (all files) 2024-11-15T16:59:41,228 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/C in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,228 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/813d4504dc5c4f2e975d1ce131b1dc2e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/4e1d8e4d01fa47bbb17471acf3cf0665, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/2beea1f69d694f329de0748c7fb67ba3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/32415ca0067a4124911b4f3da38fd389] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=47.8 K 2024-11-15T16:59:41,229 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 813d4504dc5c4f2e975d1ce131b1dc2e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731689975111 2024-11-15T16:59:41,229 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1d8e4d01fa47bbb17471acf3cf0665, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731689975763 2024-11-15T16:59:41,229 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2beea1f69d694f329de0748c7fb67ba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731689976971 2024-11-15T16:59:41,230 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 32415ca0067a4124911b4f3da38fd389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731689979169 2024-11-15T16:59:41,232 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/76fd442fe5964cf7b7f974036b23bd51 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76fd442fe5964cf7b7f974036b23bd51 2024-11-15T16:59:41,236 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/A of fec8d5aa95a6ec0fca101912a0d79069 into 76fd442fe5964cf7b7f974036b23bd51(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:41,236 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:41,236 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/A, priority=12, startTime=1731689980787; duration=0sec 2024-11-15T16:59:41,236 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:41,236 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:A 2024-11-15T16:59:41,238 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:41,238 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/0104e54635be4787a4e76f724afc7bc5 is 50, key is test_row_0/C:col10/1731689980320/Put/seqid=0 2024-11-15T16:59:41,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742205_1381 (size=12629) 2024-11-15T16:59:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T16:59:41,325 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:41,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-15T16:59:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T16:59:41,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:41,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:41,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:41,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690041386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690041386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690041405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T16:59:41,454 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/a00ae1b8c85b4a49b24894aa7bbf76fa 2024-11-15T16:59:41,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/461cb515b879426fa3a9630a0815137d is 50, key is test_row_0/B:col10/1731689980384/Put/seqid=0 2024-11-15T16:59:41,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742206_1382 (size=12151) 2024-11-15T16:59:41,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T16:59:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T16:59:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T16:59:41,649 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/0104e54635be4787a4e76f724afc7bc5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0104e54635be4787a4e76f724afc7bc5 2024-11-15T16:59:41,653 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/C of fec8d5aa95a6ec0fca101912a0d79069 into 0104e54635be4787a4e76f724afc7bc5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:41,653 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:41,653 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/C, priority=12, startTime=1731689980788; duration=0sec 2024-11-15T16:59:41,653 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:41,653 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:C 2024-11-15T16:59:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690041695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690041696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690041726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:41,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T16:59:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/461cb515b879426fa3a9630a0815137d 2024-11-15T16:59:41,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/f3ebb6eb1ba3470688833aae1de1cdd4 is 50, key is test_row_0/C:col10/1731689980384/Put/seqid=0 2024-11-15T16:59:41,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742207_1383 (size=12151) 2024-11-15T16:59:41,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T16:59:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:41,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T16:59:42,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T16:59:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:42,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:42,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690042205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:42,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690042205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:42,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T16:59:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:42,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:42,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690042246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:42,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/f3ebb6eb1ba3470688833aae1de1cdd4 2024-11-15T16:59:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/a00ae1b8c85b4a49b24894aa7bbf76fa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/a00ae1b8c85b4a49b24894aa7bbf76fa 2024-11-15T16:59:42,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/a00ae1b8c85b4a49b24894aa7bbf76fa, entries=200, sequenceid=235, filesize=38.8 K 2024-11-15T16:59:42,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/461cb515b879426fa3a9630a0815137d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/461cb515b879426fa3a9630a0815137d 2024-11-15T16:59:42,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/461cb515b879426fa3a9630a0815137d, entries=150, sequenceid=235, filesize=11.9 K 2024-11-15T16:59:42,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/f3ebb6eb1ba3470688833aae1de1cdd4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/f3ebb6eb1ba3470688833aae1de1cdd4 2024-11-15T16:59:42,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/f3ebb6eb1ba3470688833aae1de1cdd4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-15T16:59:42,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fec8d5aa95a6ec0fca101912a0d79069 in 1264ms, sequenceid=235, compaction requested=false 2024-11-15T16:59:42,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:42,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T16:59:42,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:42,399 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T16:59:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dcf81a291a1f472a852cef819c2c2296_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689981064/Put/seqid=0 2024-11-15T16:59:42,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742208_1384 (size=12304) 2024-11-15T16:59:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T16:59:42,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:42,823 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dcf81a291a1f472a852cef819c2c2296_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dcf81a291a1f472a852cef819c2c2296_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d81cd3c43d3c49e8a3d155ac60ece611, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d81cd3c43d3c49e8a3d155ac60ece611 is 175, key is test_row_0/A:col10/1731689981064/Put/seqid=0 2024-11-15T16:59:42,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742209_1385 (size=31105) 2024-11-15T16:59:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:43,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:43,231 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d81cd3c43d3c49e8a3d155ac60ece611 2024-11-15T16:59:43,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/97c2b9355b20441b92eeb61a244b6800 is 50, key is test_row_0/B:col10/1731689981064/Put/seqid=0 2024-11-15T16:59:43,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742210_1386 (size=12151) 2024-11-15T16:59:43,244 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/97c2b9355b20441b92eeb61a244b6800 2024-11-15T16:59:43,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/39afc855fe7444b7b5585f7606e4adfb is 50, key is test_row_0/C:col10/1731689981064/Put/seqid=0 2024-11-15T16:59:43,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742211_1387 (size=12151) 2024-11-15T16:59:43,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690043315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690043316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690043316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56058 deadline: 1731690043335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,337 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in fec8d5aa95a6ec0fca101912a0d79069 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56046 deadline: 1731690043346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,349 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in fec8d5aa95a6ec0fca101912a0d79069 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T16:59:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690043435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690043435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690043436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T16:59:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690043645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,655 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/39afc855fe7444b7b5585f7606e4adfb 2024-11-15T16:59:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690043646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690043646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/d81cd3c43d3c49e8a3d155ac60ece611 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d81cd3c43d3c49e8a3d155ac60ece611 2024-11-15T16:59:43,664 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d81cd3c43d3c49e8a3d155ac60ece611, entries=150, sequenceid=249, filesize=30.4 K 2024-11-15T16:59:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/97c2b9355b20441b92eeb61a244b6800 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/97c2b9355b20441b92eeb61a244b6800 2024-11-15T16:59:43,669 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/97c2b9355b20441b92eeb61a244b6800, entries=150, sequenceid=249, filesize=11.9 K 2024-11-15T16:59:43,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/39afc855fe7444b7b5585f7606e4adfb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/39afc855fe7444b7b5585f7606e4adfb 2024-11-15T16:59:43,675 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/39afc855fe7444b7b5585f7606e4adfb, entries=150, sequenceid=249, filesize=11.9 K 2024-11-15T16:59:43,676 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fec8d5aa95a6ec0fca101912a0d79069 in 1277ms, sequenceid=249, compaction requested=true 2024-11-15T16:59:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-15T16:59:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-15T16:59:43,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T16:59:43,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-11-15T16:59:43,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.3530 sec 2024-11-15T16:59:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:43,959 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:43,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T16:59:43,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b562933e5184496f872c1a85d5e38829_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689983314/Put/seqid=0 2024-11-15T16:59:43,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742212_1388 (size=14994) 2024-11-15T16:59:43,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690043985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:43,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690043986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690043987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690044106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690044106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690044106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690044316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690044316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690044325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,372 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:44,376 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b562933e5184496f872c1a85d5e38829_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b562933e5184496f872c1a85d5e38829_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:44,377 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0241c0c409aa4d4294deb1819368f63f, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:44,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0241c0c409aa4d4294deb1819368f63f is 175, key is test_row_0/A:col10/1731689983314/Put/seqid=0 2024-11-15T16:59:44,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742213_1389 (size=39949) 2024-11-15T16:59:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690044636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690044636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690044636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:44,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0241c0c409aa4d4294deb1819368f63f 2024-11-15T16:59:44,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/d3dfaa587ee04262885acf295ebdd37a is 50, key is test_row_0/B:col10/1731689983314/Put/seqid=0 2024-11-15T16:59:44,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742214_1390 (size=12301) 2024-11-15T16:59:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690045146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690045146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690045146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:45,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/d3dfaa587ee04262885acf295ebdd37a 2024-11-15T16:59:45,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/e83a42e8bac340f3b6e699f63ea696d4 is 50, key is test_row_0/C:col10/1731689983314/Put/seqid=0 2024-11-15T16:59:45,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742215_1391 (size=12301) 2024-11-15T16:59:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T16:59:45,496 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:45,496 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-15T16:59:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T16:59:45,498 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:45,499 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:45,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T16:59:45,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/e83a42e8bac340f3b6e699f63ea696d4 2024-11-15T16:59:45,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/0241c0c409aa4d4294deb1819368f63f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0241c0c409aa4d4294deb1819368f63f 2024-11-15T16:59:45,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0241c0c409aa4d4294deb1819368f63f, entries=200, sequenceid=275, filesize=39.0 K 2024-11-15T16:59:45,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/d3dfaa587ee04262885acf295ebdd37a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/d3dfaa587ee04262885acf295ebdd37a 2024-11-15T16:59:45,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/d3dfaa587ee04262885acf295ebdd37a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-15T16:59:45,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/e83a42e8bac340f3b6e699f63ea696d4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e83a42e8bac340f3b6e699f63ea696d4 2024-11-15T16:59:45,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e83a42e8bac340f3b6e699f63ea696d4, entries=150, sequenceid=275, filesize=12.0 K 2024-11-15T16:59:45,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fec8d5aa95a6ec0fca101912a0d79069 in 1673ms, sequenceid=275, compaction requested=true 2024-11-15T16:59:45,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:45,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:45,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:45,633 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:45,633 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:45,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:45,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:45,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:45,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:45,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:45,635 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:45,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/A is initiating minor compaction (all files) 2024-11-15T16:59:45,635 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/B is initiating minor compaction (all files) 2024-11-15T16:59:45,635 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/A in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:45,635 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/B in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:45,635 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76fd442fe5964cf7b7f974036b23bd51, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/a00ae1b8c85b4a49b24894aa7bbf76fa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d81cd3c43d3c49e8a3d155ac60ece611, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0241c0c409aa4d4294deb1819368f63f] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=139.0 K 2024-11-15T16:59:45,635 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/ae616b5871124b7ea9087f9e6fceece4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/461cb515b879426fa3a9630a0815137d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/97c2b9355b20441b92eeb61a244b6800, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/d3dfaa587ee04262885acf295ebdd37a] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=48.1 K 2024-11-15T16:59:45,635 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:45,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76fd442fe5964cf7b7f974036b23bd51, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/a00ae1b8c85b4a49b24894aa7bbf76fa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d81cd3c43d3c49e8a3d155ac60ece611, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0241c0c409aa4d4294deb1819368f63f] 2024-11-15T16:59:45,635 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76fd442fe5964cf7b7f974036b23bd51, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731689979169 2024-11-15T16:59:45,635 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting ae616b5871124b7ea9087f9e6fceece4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731689979169 2024-11-15T16:59:45,636 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 461cb515b879426fa3a9630a0815137d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731689980380 2024-11-15T16:59:45,636 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00ae1b8c85b4a49b24894aa7bbf76fa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731689980380 2024-11-15T16:59:45,636 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 97c2b9355b20441b92eeb61a244b6800, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731689981046 2024-11-15T16:59:45,636 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d81cd3c43d3c49e8a3d155ac60ece611, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731689981046 2024-11-15T16:59:45,636 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d3dfaa587ee04262885acf295ebdd37a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731689983304 2024-11-15T16:59:45,636 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0241c0c409aa4d4294deb1819368f63f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731689983300 2024-11-15T16:59:45,644 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:45,645 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:45,646 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ac66fa11b7af40ee9f48f5d6acc46f0f_fec8d5aa95a6ec0fca101912a0d79069 store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:45,646 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/8ae4e26fada64854b5dc4ce81728aead is 50, key is test_row_0/B:col10/1731689983314/Put/seqid=0 2024-11-15T16:59:45,648 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ac66fa11b7af40ee9f48f5d6acc46f0f_fec8d5aa95a6ec0fca101912a0d79069, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:45,648 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ac66fa11b7af40ee9f48f5d6acc46f0f_fec8d5aa95a6ec0fca101912a0d79069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:45,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T16:59:45,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:45,652 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T16:59:45,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742216_1392 (size=12915) 2024-11-15T16:59:45,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742217_1393 (size=4469) 2024-11-15T16:59:45,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115048cb8cbc8b14923855792c97b0ae876_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689983974/Put/seqid=0 2024-11-15T16:59:45,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742218_1394 (size=12454) 2024-11-15T16:59:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T16:59:46,065 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/8ae4e26fada64854b5dc4ce81728aead as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8ae4e26fada64854b5dc4ce81728aead 2024-11-15T16:59:46,070 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/B of fec8d5aa95a6ec0fca101912a0d79069 into 8ae4e26fada64854b5dc4ce81728aead(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:46,070 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:46,070 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/B, priority=12, startTime=1731689985633; duration=0sec 2024-11-15T16:59:46,070 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:46,070 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:B 2024-11-15T16:59:46,071 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T16:59:46,072 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:46,073 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T16:59:46,073 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/C is initiating minor compaction (all files) 2024-11-15T16:59:46,073 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/C in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:46,073 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0104e54635be4787a4e76f724afc7bc5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/f3ebb6eb1ba3470688833aae1de1cdd4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/39afc855fe7444b7b5585f7606e4adfb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e83a42e8bac340f3b6e699f63ea696d4] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=48.1 K 2024-11-15T16:59:46,073 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/fd74fc4dbb784439a30ec8d223ec2f85 is 175, key is test_row_0/A:col10/1731689983314/Put/seqid=0 2024-11-15T16:59:46,074 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 0104e54635be4787a4e76f724afc7bc5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731689979169 2024-11-15T16:59:46,074 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ebb6eb1ba3470688833aae1de1cdd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731689980380 2024-11-15T16:59:46,075 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 39afc855fe7444b7b5585f7606e4adfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731689981046 2024-11-15T16:59:46,075 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e83a42e8bac340f3b6e699f63ea696d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731689983304 2024-11-15T16:59:46,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742219_1395 (size=31869) 2024-11-15T16:59:46,082 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:46,084 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/641f5483fd3c4afa9ec578a88b512fd7 is 50, key is test_row_0/C:col10/1731689983314/Put/seqid=0 2024-11-15T16:59:46,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115048cb8cbc8b14923855792c97b0ae876_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115048cb8cbc8b14923855792c97b0ae876_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:46,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/82225ce07341442b97f38992c755d6b3, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:46,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/82225ce07341442b97f38992c755d6b3 is 175, key is test_row_0/A:col10/1731689983974/Put/seqid=0 2024-11-15T16:59:46,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742220_1396 (size=12915) 2024-11-15T16:59:46,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742221_1397 (size=31255) 2024-11-15T16:59:46,100 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/641f5483fd3c4afa9ec578a88b512fd7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/641f5483fd3c4afa9ec578a88b512fd7 2024-11-15T16:59:46,107 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/C of fec8d5aa95a6ec0fca101912a0d79069 into 641f5483fd3c4afa9ec578a88b512fd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:46,107 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:46,107 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/C, priority=12, startTime=1731689985634; duration=0sec 2024-11-15T16:59:46,107 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:46,107 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:C 2024-11-15T16:59:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T16:59:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:46,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:46,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690046259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690046264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690046267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690046376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690046376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690046385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,487 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/fd74fc4dbb784439a30ec8d223ec2f85 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/fd74fc4dbb784439a30ec8d223ec2f85 2024-11-15T16:59:46,496 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/82225ce07341442b97f38992c755d6b3 2024-11-15T16:59:46,496 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/A of fec8d5aa95a6ec0fca101912a0d79069 into fd74fc4dbb784439a30ec8d223ec2f85(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:46,496 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:46,496 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/A, priority=12, startTime=1731689985633; duration=0sec 2024-11-15T16:59:46,497 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:46,497 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:A 2024-11-15T16:59:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/16077628de134d6a8d3d0154a3da385b is 50, key is test_row_0/B:col10/1731689983974/Put/seqid=0 2024-11-15T16:59:46,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742222_1398 (size=12301) 2024-11-15T16:59:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690046586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690046586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690046595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T16:59:46,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690046896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690046906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,908 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/16077628de134d6a8d3d0154a3da385b 2024-11-15T16:59:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690046906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:46,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/d39bca495bf84e8ba619b2a15a5f6448 is 50, key is test_row_0/C:col10/1731689983974/Put/seqid=0 2024-11-15T16:59:46,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742223_1399 (size=12301) 2024-11-15T16:59:47,320 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/d39bca495bf84e8ba619b2a15a5f6448 2024-11-15T16:59:47,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/82225ce07341442b97f38992c755d6b3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/82225ce07341442b97f38992c755d6b3 2024-11-15T16:59:47,328 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/82225ce07341442b97f38992c755d6b3, entries=150, sequenceid=285, filesize=30.5 K 2024-11-15T16:59:47,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/16077628de134d6a8d3d0154a3da385b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/16077628de134d6a8d3d0154a3da385b 2024-11-15T16:59:47,332 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/16077628de134d6a8d3d0154a3da385b, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T16:59:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/d39bca495bf84e8ba619b2a15a5f6448 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/d39bca495bf84e8ba619b2a15a5f6448 2024-11-15T16:59:47,336 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/d39bca495bf84e8ba619b2a15a5f6448, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T16:59:47,337 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for fec8d5aa95a6ec0fca101912a0d79069 in 1685ms, sequenceid=285, compaction requested=false 2024-11-15T16:59:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-15T16:59:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-15T16:59:47,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T16:59:47,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-11-15T16:59:47,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8440 sec 2024-11-15T16:59:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:47,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:47,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T16:59:47,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115af01570d9f3f4225a9d66c44e4b9e3ef_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689986262/Put/seqid=0 2024-11-15T16:59:47,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742224_1400 (size=14994) 2024-11-15T16:59:47,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:47,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115af01570d9f3f4225a9d66c44e4b9e3ef_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115af01570d9f3f4225a9d66c44e4b9e3ef_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:47,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/bdeeaf4f1c0241929b7a49bc30253b5b, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:47,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/bdeeaf4f1c0241929b7a49bc30253b5b is 175, key is test_row_0/A:col10/1731689986262/Put/seqid=0 2024-11-15T16:59:47,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690047425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690047430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742225_1401 (size=39949) 2024-11-15T16:59:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690047434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690047545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690047545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690047556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T16:59:47,646 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:47,646 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-15T16:59:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T16:59:47,648 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:47,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:47,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:47,694 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-11-15T16:59:47,694 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:47,694 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,694 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:47,698 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-15T16:59:47,698 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:47,698 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,698 INFO [Thread-1480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1480. 2024-11-15T16:59:47,698 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,698 DEBUG [Thread-1480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:47,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:47,698 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:47,698 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,702 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-15T16:59:47,702 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:47,702 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,702 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:47,705 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-11-15T16:59:47,705 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:47,705 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,705 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:47,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T16:59:47,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690047756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690047755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690047766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:47,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T16:59:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:47,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:47,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/bdeeaf4f1c0241929b7a49bc30253b5b 2024-11-15T16:59:47,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/1cf4ddd33a384417938bd0b7fd284aca is 50, key is test_row_0/B:col10/1731689986262/Put/seqid=0 2024-11-15T16:59:47,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742226_1402 (size=12301) 2024-11-15T16:59:47,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T16:59:47,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:47,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:47,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:47,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:47,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T16:59:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690048066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690048066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690048075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:48,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T16:59:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/1cf4ddd33a384417938bd0b7fd284aca 2024-11-15T16:59:48,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/20d868b068d14d6f9c428e09bd4f4702 is 50, key is test_row_0/C:col10/1731689986262/Put/seqid=0 2024-11-15T16:59:48,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742227_1403 (size=12301) 2024-11-15T16:59:48,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T16:59:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T16:59:48,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T16:59:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T16:59:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:48,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56044 deadline: 1731690048576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:48,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56068 deadline: 1731690048576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56080 deadline: 1731690048586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T16:59:48,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/20d868b068d14d6f9c428e09bd4f4702 2024-11-15T16:59:48,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/bdeeaf4f1c0241929b7a49bc30253b5b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/bdeeaf4f1c0241929b7a49bc30253b5b 2024-11-15T16:59:48,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/bdeeaf4f1c0241929b7a49bc30253b5b, entries=200, sequenceid=315, filesize=39.0 K 2024-11-15T16:59:48,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/1cf4ddd33a384417938bd0b7fd284aca as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1cf4ddd33a384417938bd0b7fd284aca 2024-11-15T16:59:48,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1cf4ddd33a384417938bd0b7fd284aca, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T16:59:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/20d868b068d14d6f9c428e09bd4f4702 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/20d868b068d14d6f9c428e09bd4f4702 2024-11-15T16:59:48,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/20d868b068d14d6f9c428e09bd4f4702, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T16:59:48,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for fec8d5aa95a6ec0fca101912a0d79069 in 1256ms, sequenceid=315, compaction requested=true 2024-11-15T16:59:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T16:59:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:48,669 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T16:59:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fec8d5aa95a6ec0fca101912a0d79069:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T16:59:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:48,669 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:48,669 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:48,669 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:48,669 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/A is initiating minor compaction (all files) 2024-11-15T16:59:48,669 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/B is initiating minor compaction (all files) 2024-11-15T16:59:48,669 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/A in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,669 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/B in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,669 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/fd74fc4dbb784439a30ec8d223ec2f85, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/82225ce07341442b97f38992c755d6b3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/bdeeaf4f1c0241929b7a49bc30253b5b] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=100.7 K 2024-11-15T16:59:48,670 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8ae4e26fada64854b5dc4ce81728aead, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/16077628de134d6a8d3d0154a3da385b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1cf4ddd33a384417938bd0b7fd284aca] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=36.6 K 2024-11-15T16:59:48,669 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,670 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/fd74fc4dbb784439a30ec8d223ec2f85, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/82225ce07341442b97f38992c755d6b3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/bdeeaf4f1c0241929b7a49bc30253b5b] 2024-11-15T16:59:48,670 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae4e26fada64854b5dc4ce81728aead, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731689983304 2024-11-15T16:59:48,670 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd74fc4dbb784439a30ec8d223ec2f85, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731689983304 2024-11-15T16:59:48,670 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 16077628de134d6a8d3d0154a3da385b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731689983974 2024-11-15T16:59:48,670 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82225ce07341442b97f38992c755d6b3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731689983974 2024-11-15T16:59:48,670 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cf4ddd33a384417938bd0b7fd284aca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731689986262 2024-11-15T16:59:48,670 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdeeaf4f1c0241929b7a49bc30253b5b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731689986254 2024-11-15T16:59:48,676 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:48,676 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:48,677 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/a8955c498fb640058e102a69bed9cd58 is 50, key is test_row_0/B:col10/1731689986262/Put/seqid=0 2024-11-15T16:59:48,678 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115978ebbb6bac347f2b51d25ad78a52c65_fec8d5aa95a6ec0fca101912a0d79069 store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:48,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742228_1404 (size=13017) 2024-11-15T16:59:48,703 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115978ebbb6bac347f2b51d25ad78a52c65_fec8d5aa95a6ec0fca101912a0d79069, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:48,703 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115978ebbb6bac347f2b51d25ad78a52c65_fec8d5aa95a6ec0fca101912a0d79069 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:48,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742229_1405 (size=4469) 2024-11-15T16:59:48,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T16:59:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:48,719 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T16:59:48,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec167ed4556946dcab591aa774c86d7e_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_0/A:col10/1731689987432/Put/seqid=0 2024-11-15T16:59:48,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742230_1406 (size=12454) 2024-11-15T16:59:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T16:59:49,084 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/a8955c498fb640058e102a69bed9cd58 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/a8955c498fb640058e102a69bed9cd58 2024-11-15T16:59:49,088 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/B of fec8d5aa95a6ec0fca101912a0d79069 into a8955c498fb640058e102a69bed9cd58(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:49,088 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:49,088 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/B, priority=13, startTime=1731689988669; duration=0sec 2024-11-15T16:59:49,088 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T16:59:49,088 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:B 2024-11-15T16:59:49,088 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T16:59:49,089 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T16:59:49,089 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): fec8d5aa95a6ec0fca101912a0d79069/C is initiating minor compaction (all files) 2024-11-15T16:59:49,089 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fec8d5aa95a6ec0fca101912a0d79069/C in TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:49,089 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/641f5483fd3c4afa9ec578a88b512fd7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/d39bca495bf84e8ba619b2a15a5f6448, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/20d868b068d14d6f9c428e09bd4f4702] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp, totalSize=36.6 K 2024-11-15T16:59:49,089 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 641f5483fd3c4afa9ec578a88b512fd7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731689983304 2024-11-15T16:59:49,089 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d39bca495bf84e8ba619b2a15a5f6448, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731689983974 2024-11-15T16:59:49,090 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d868b068d14d6f9c428e09bd4f4702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731689986262 2024-11-15T16:59:49,096 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:49,096 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/677f95cf87654d8c939dafda0418e3ec is 50, key is test_row_0/C:col10/1731689986262/Put/seqid=0 2024-11-15T16:59:49,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742231_1407 (size=13017) 2024-11-15T16:59:49,107 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fec8d5aa95a6ec0fca101912a0d79069#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T16:59:49,108 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/f2a79541d3b14d6282e762e7ac5dab26 is 175, key is test_row_0/A:col10/1731689986262/Put/seqid=0 2024-11-15T16:59:49,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742232_1408 (size=31971) 2024-11-15T16:59:49,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:49,137 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec167ed4556946dcab591aa774c86d7e_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec167ed4556946dcab591aa774c86d7e_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:49,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/8a3e98a24af74fe5b3b08dd92e455f8f, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:49,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/8a3e98a24af74fe5b3b08dd92e455f8f is 175, key is test_row_0/A:col10/1731689987432/Put/seqid=0 2024-11-15T16:59:49,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742233_1409 (size=31255) 2024-11-15T16:59:49,504 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/677f95cf87654d8c939dafda0418e3ec as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/677f95cf87654d8c939dafda0418e3ec 2024-11-15T16:59:49,507 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/C of fec8d5aa95a6ec0fca101912a0d79069 into 677f95cf87654d8c939dafda0418e3ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:49,507 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:49,508 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/C, priority=13, startTime=1731689988669; duration=0sec 2024-11-15T16:59:49,508 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:49,508 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:C 2024-11-15T16:59:49,515 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/f2a79541d3b14d6282e762e7ac5dab26 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/f2a79541d3b14d6282e762e7ac5dab26 2024-11-15T16:59:49,518 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fec8d5aa95a6ec0fca101912a0d79069/A of fec8d5aa95a6ec0fca101912a0d79069 into f2a79541d3b14d6282e762e7ac5dab26(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T16:59:49,518 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:49,518 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., storeName=fec8d5aa95a6ec0fca101912a0d79069/A, priority=13, startTime=1731689988668; duration=0sec 2024-11-15T16:59:49,518 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T16:59:49,518 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fec8d5aa95a6ec0fca101912a0d79069:A 2024-11-15T16:59:49,542 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/8a3e98a24af74fe5b3b08dd92e455f8f 2024-11-15T16:59:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/fedebda8fa244990939958135e126a0c is 50, key is test_row_0/B:col10/1731689987432/Put/seqid=0 2024-11-15T16:59:49,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742234_1410 (size=12301) 2024-11-15T16:59:49,551 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/fedebda8fa244990939958135e126a0c 2024-11-15T16:59:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/42630d3f8c684e158a411fae7cefd0f3 is 50, key is test_row_0/C:col10/1731689987432/Put/seqid=0 2024-11-15T16:59:49,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742235_1411 (size=12301) 2024-11-15T16:59:49,587 INFO [Thread-1467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1467. 2024-11-15T16:59:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:49,588 DEBUG [Thread-1467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:49,588 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:49,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. as already flushing 2024-11-15T16:59:49,588 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:49,588 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:49,588 INFO [Thread-1463 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1463. 2024-11-15T16:59:49,588 DEBUG [Thread-1463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:49,588 DEBUG [Thread-1463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:49,588 DEBUG [Thread-1463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:49,588 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:49,597 INFO [Thread-1471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1471. 2024-11-15T16:59:49,597 DEBUG [Thread-1471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:49,597 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:49,597 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:49,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T16:59:49,960 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/42630d3f8c684e158a411fae7cefd0f3 2024-11-15T16:59:49,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/8a3e98a24af74fe5b3b08dd92e455f8f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/8a3e98a24af74fe5b3b08dd92e455f8f 2024-11-15T16:59:49,966 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/8a3e98a24af74fe5b3b08dd92e455f8f, entries=150, sequenceid=324, filesize=30.5 K 2024-11-15T16:59:49,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/fedebda8fa244990939958135e126a0c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/fedebda8fa244990939958135e126a0c 2024-11-15T16:59:49,970 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/fedebda8fa244990939958135e126a0c, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T16:59:49,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/42630d3f8c684e158a411fae7cefd0f3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/42630d3f8c684e158a411fae7cefd0f3 2024-11-15T16:59:49,973 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/42630d3f8c684e158a411fae7cefd0f3, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T16:59:49,974 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=20.13 KB/20610 for fec8d5aa95a6ec0fca101912a0d79069 in 1255ms, sequenceid=324, compaction requested=false 2024-11-15T16:59:49,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for fec8d5aa95a6ec0fca101912a0d79069: 2024-11-15T16:59:49,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:49,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-15T16:59:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-15T16:59:49,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T16:59:49,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3260 sec 2024-11-15T16:59:49,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.3310 sec 2024-11-15T16:59:50,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T16:59:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T16:59:51,826 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:53,428 INFO [Thread-1469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1469. 2024-11-15T16:59:53,428 INFO [Thread-1465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1465. 2024-11-15T16:59:53,428 DEBUG [Thread-1465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:53,428 DEBUG [Thread-1469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T16:59:53,428 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:53,428 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:53,428 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:53,428 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:53,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:53,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1354 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4057 rows 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1361 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4080 rows 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1353 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4058 rows 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1358 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4067 rows 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1359 2024-11-15T16:59:53,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4070 rows 2024-11-15T16:59:53,429 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T16:59:53,429 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T16:59:53,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:53,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:53,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T16:59:53,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T16:59:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T16:59:53,432 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689993432"}]},"ts":"1731689993432"} 2024-11-15T16:59:53,433 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T16:59:53,433 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T16:59:53,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T16:59:53,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, UNASSIGN}] 2024-11-15T16:59:53,436 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, UNASSIGN 2024-11-15T16:59:53,436 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:53,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, UNASSIGN because future has completed 2024-11-15T16:59:53,437 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T16:59:53,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:59:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T16:59:53,589 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:53,589 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T16:59:53,589 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing fec8d5aa95a6ec0fca101912a0d79069, disabling compactions & flushes 2024-11-15T16:59:53,589 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:53,589 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:53,589 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. after waiting 0 ms 2024-11-15T16:59:53,589 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:53,590 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing fec8d5aa95a6ec0fca101912a0d79069 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T16:59:53,595 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111564375233df9748a992510dc49a601ac4_fec8d5aa95a6ec0fca101912a0d79069 is 50, key is test_row_2/A:col10/1731689993426/Put/seqid=0 2024-11-15T16:59:53,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742236_1412 (size=7374) 2024-11-15T16:59:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T16:59:53,999 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:54,002 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111564375233df9748a992510dc49a601ac4_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111564375233df9748a992510dc49a601ac4_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:54,003 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e128fdca340947a5b39bac02ccfa56f2, store: [table=TestAcidGuarantees family=A region=fec8d5aa95a6ec0fca101912a0d79069] 2024-11-15T16:59:54,003 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e128fdca340947a5b39bac02ccfa56f2 is 175, key is test_row_2/A:col10/1731689993426/Put/seqid=0 2024-11-15T16:59:54,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742237_1413 (size=13865) 2024-11-15T16:59:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T16:59:54,407 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e128fdca340947a5b39bac02ccfa56f2 2024-11-15T16:59:54,413 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/53f992ab2ebc4c2db522dc14be91786e is 50, key is test_row_2/B:col10/1731689993426/Put/seqid=0 2024-11-15T16:59:54,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742238_1414 (size=7415) 2024-11-15T16:59:54,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/9002d20928ba4cfb8a155ccf38cdf94b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b59e259c86a14f0aaf6688ecdd66e5bf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d6cb6bd9c619456097be8c8d494a68ae, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/baa27d7d2f1141b886532352083a1788, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2bbc526c11634dc2b9f348f99fac59a7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b7cd2dcf834a4204843e319a37d627fe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0db67e22bfb64fe88dc34a840c13b4a3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/aa9b2d3cb17a43c28407ed4b6ff533f6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e33a19c4e3e4476f893db358827cdbf0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/19d760ae2e7e4f8fb6f15a465194b780, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2b604dd0a3484e9196268cdd5c10c365, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/7d926eda42aa4e95909124197748cfd0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76616264647d430da232be5cc1a47afc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/1e108181ce8c430db087c0e2c082d986, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76fd442fe5964cf7b7f974036b23bd51, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/a00ae1b8c85b4a49b24894aa7bbf76fa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d81cd3c43d3c49e8a3d155ac60ece611, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0241c0c409aa4d4294deb1819368f63f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/fd74fc4dbb784439a30ec8d223ec2f85, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/82225ce07341442b97f38992c755d6b3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/bdeeaf4f1c0241929b7a49bc30253b5b] to archive 2024-11-15T16:59:54,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:59:54,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/9002d20928ba4cfb8a155ccf38cdf94b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/9002d20928ba4cfb8a155ccf38cdf94b 2024-11-15T16:59:54,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b59e259c86a14f0aaf6688ecdd66e5bf to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b59e259c86a14f0aaf6688ecdd66e5bf 2024-11-15T16:59:54,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d6cb6bd9c619456097be8c8d494a68ae to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d6cb6bd9c619456097be8c8d494a68ae 2024-11-15T16:59:54,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/baa27d7d2f1141b886532352083a1788 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/baa27d7d2f1141b886532352083a1788 2024-11-15T16:59:54,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2bbc526c11634dc2b9f348f99fac59a7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2bbc526c11634dc2b9f348f99fac59a7 2024-11-15T16:59:54,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b7cd2dcf834a4204843e319a37d627fe to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/b7cd2dcf834a4204843e319a37d627fe 2024-11-15T16:59:54,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0db67e22bfb64fe88dc34a840c13b4a3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0db67e22bfb64fe88dc34a840c13b4a3 2024-11-15T16:59:54,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/aa9b2d3cb17a43c28407ed4b6ff533f6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/aa9b2d3cb17a43c28407ed4b6ff533f6 2024-11-15T16:59:54,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e33a19c4e3e4476f893db358827cdbf0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e33a19c4e3e4476f893db358827cdbf0 2024-11-15T16:59:54,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/19d760ae2e7e4f8fb6f15a465194b780 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/19d760ae2e7e4f8fb6f15a465194b780 2024-11-15T16:59:54,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2b604dd0a3484e9196268cdd5c10c365 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/2b604dd0a3484e9196268cdd5c10c365 2024-11-15T16:59:54,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/7d926eda42aa4e95909124197748cfd0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/7d926eda42aa4e95909124197748cfd0 2024-11-15T16:59:54,560 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76616264647d430da232be5cc1a47afc to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76616264647d430da232be5cc1a47afc 2024-11-15T16:59:54,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/1e108181ce8c430db087c0e2c082d986 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/1e108181ce8c430db087c0e2c082d986 2024-11-15T16:59:54,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76fd442fe5964cf7b7f974036b23bd51 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/76fd442fe5964cf7b7f974036b23bd51 2024-11-15T16:59:54,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/a00ae1b8c85b4a49b24894aa7bbf76fa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/a00ae1b8c85b4a49b24894aa7bbf76fa 2024-11-15T16:59:54,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d81cd3c43d3c49e8a3d155ac60ece611 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/d81cd3c43d3c49e8a3d155ac60ece611 2024-11-15T16:59:54,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0241c0c409aa4d4294deb1819368f63f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/0241c0c409aa4d4294deb1819368f63f 2024-11-15T16:59:54,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/fd74fc4dbb784439a30ec8d223ec2f85 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/fd74fc4dbb784439a30ec8d223ec2f85 2024-11-15T16:59:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T16:59:54,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/82225ce07341442b97f38992c755d6b3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/82225ce07341442b97f38992c755d6b3 2024-11-15T16:59:54,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/bdeeaf4f1c0241929b7a49bc30253b5b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/bdeeaf4f1c0241929b7a49bc30253b5b 2024-11-15T16:59:54,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/19549cb6c66b4b01903485eb76e5a27d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/323372814e5e4d588c7a7f87d810e912, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/f75816607a4d4d75aeb47d3aaf071b4d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e038d9330a9e4b36bd356d4d69239d94, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/b6aed81760c146ec965b5a427f74fc64, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e81bff7ea38d4dc482b06ea4eb534920, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/196c983bf6044ac298efa63eb0c8f29d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/9efe4696d7ee482a8e2764c1384a6017, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/85c4f15ec7c34b7aba90d2d1ae928c7c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/77f02daa19764f7ebfba1bc9075a9d8c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8a9a49d61c3347e8a311787e36db8b62, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/404d8ccf35434f9c98543646e8a7c04a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/ae616b5871124b7ea9087f9e6fceece4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1757ec1013034fef8578be862d29d173, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/461cb515b879426fa3a9630a0815137d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/97c2b9355b20441b92eeb61a244b6800, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8ae4e26fada64854b5dc4ce81728aead, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/d3dfaa587ee04262885acf295ebdd37a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/16077628de134d6a8d3d0154a3da385b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1cf4ddd33a384417938bd0b7fd284aca] to archive 2024-11-15T16:59:54,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:59:54,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/19549cb6c66b4b01903485eb76e5a27d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/19549cb6c66b4b01903485eb76e5a27d 2024-11-15T16:59:54,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/323372814e5e4d588c7a7f87d810e912 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/323372814e5e4d588c7a7f87d810e912 2024-11-15T16:59:54,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/f75816607a4d4d75aeb47d3aaf071b4d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/f75816607a4d4d75aeb47d3aaf071b4d 2024-11-15T16:59:54,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/06bb4d1dfe9f4a5a99f008d2bbf9bfcc 2024-11-15T16:59:54,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e038d9330a9e4b36bd356d4d69239d94 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e038d9330a9e4b36bd356d4d69239d94 2024-11-15T16:59:54,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/b6aed81760c146ec965b5a427f74fc64 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/b6aed81760c146ec965b5a427f74fc64 2024-11-15T16:59:54,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e81bff7ea38d4dc482b06ea4eb534920 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/e81bff7ea38d4dc482b06ea4eb534920 2024-11-15T16:59:54,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/196c983bf6044ac298efa63eb0c8f29d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/196c983bf6044ac298efa63eb0c8f29d 2024-11-15T16:59:54,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/9efe4696d7ee482a8e2764c1384a6017 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/9efe4696d7ee482a8e2764c1384a6017 2024-11-15T16:59:54,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/85c4f15ec7c34b7aba90d2d1ae928c7c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/85c4f15ec7c34b7aba90d2d1ae928c7c 2024-11-15T16:59:54,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/77f02daa19764f7ebfba1bc9075a9d8c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/77f02daa19764f7ebfba1bc9075a9d8c 2024-11-15T16:59:54,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8a9a49d61c3347e8a311787e36db8b62 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8a9a49d61c3347e8a311787e36db8b62 2024-11-15T16:59:54,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/404d8ccf35434f9c98543646e8a7c04a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/404d8ccf35434f9c98543646e8a7c04a 2024-11-15T16:59:54,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/ae616b5871124b7ea9087f9e6fceece4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/ae616b5871124b7ea9087f9e6fceece4 2024-11-15T16:59:54,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1757ec1013034fef8578be862d29d173 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1757ec1013034fef8578be862d29d173 2024-11-15T16:59:54,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/461cb515b879426fa3a9630a0815137d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/461cb515b879426fa3a9630a0815137d 2024-11-15T16:59:54,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/97c2b9355b20441b92eeb61a244b6800 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/97c2b9355b20441b92eeb61a244b6800 2024-11-15T16:59:54,583 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8ae4e26fada64854b5dc4ce81728aead to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/8ae4e26fada64854b5dc4ce81728aead 2024-11-15T16:59:54,584 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/d3dfaa587ee04262885acf295ebdd37a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/d3dfaa587ee04262885acf295ebdd37a 2024-11-15T16:59:54,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/16077628de134d6a8d3d0154a3da385b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/16077628de134d6a8d3d0154a3da385b 2024-11-15T16:59:54,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1cf4ddd33a384417938bd0b7fd284aca to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/1cf4ddd33a384417938bd0b7fd284aca 2024-11-15T16:59:54,588 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/52df3cabc4104737a3a19d43edfe3688, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/474672473dd948faa7494089fe1b8799, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/1c7a754dcc794796bd9e6da94532e54f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e168d4e69c0c45c389da6b5ed29c7044, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/5fb07e3ec6234d7886bebf5c7811ec3f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/6bf03b6f6cfe4f109c1c0dc287c7df6e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/212e073fe955481789e9d079bd9032b5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0e4a3d4fa38b459b96f94576061b34ec, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/62e8af28df8e461c8f1c930e5de98ac6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/813d4504dc5c4f2e975d1ce131b1dc2e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/48d5cabba75541d9a80c324a2508ee98, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/4e1d8e4d01fa47bbb17471acf3cf0665, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/2beea1f69d694f329de0748c7fb67ba3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0104e54635be4787a4e76f724afc7bc5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/32415ca0067a4124911b4f3da38fd389, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/f3ebb6eb1ba3470688833aae1de1cdd4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/39afc855fe7444b7b5585f7606e4adfb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/641f5483fd3c4afa9ec578a88b512fd7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e83a42e8bac340f3b6e699f63ea696d4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/d39bca495bf84e8ba619b2a15a5f6448, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/20d868b068d14d6f9c428e09bd4f4702] to archive 2024-11-15T16:59:54,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T16:59:54,590 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/52df3cabc4104737a3a19d43edfe3688 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/52df3cabc4104737a3a19d43edfe3688 2024-11-15T16:59:54,590 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/474672473dd948faa7494089fe1b8799 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/474672473dd948faa7494089fe1b8799 2024-11-15T16:59:54,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/1c7a754dcc794796bd9e6da94532e54f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/1c7a754dcc794796bd9e6da94532e54f 2024-11-15T16:59:54,592 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e168d4e69c0c45c389da6b5ed29c7044 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e168d4e69c0c45c389da6b5ed29c7044 2024-11-15T16:59:54,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/5fb07e3ec6234d7886bebf5c7811ec3f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/5fb07e3ec6234d7886bebf5c7811ec3f 2024-11-15T16:59:54,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/6bf03b6f6cfe4f109c1c0dc287c7df6e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/6bf03b6f6cfe4f109c1c0dc287c7df6e 2024-11-15T16:59:54,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/212e073fe955481789e9d079bd9032b5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/212e073fe955481789e9d079bd9032b5 2024-11-15T16:59:54,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0e4a3d4fa38b459b96f94576061b34ec to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0e4a3d4fa38b459b96f94576061b34ec 2024-11-15T16:59:54,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/62e8af28df8e461c8f1c930e5de98ac6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/62e8af28df8e461c8f1c930e5de98ac6 2024-11-15T16:59:54,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/813d4504dc5c4f2e975d1ce131b1dc2e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/813d4504dc5c4f2e975d1ce131b1dc2e 2024-11-15T16:59:54,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/48d5cabba75541d9a80c324a2508ee98 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/48d5cabba75541d9a80c324a2508ee98 2024-11-15T16:59:54,598 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/4e1d8e4d01fa47bbb17471acf3cf0665 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/4e1d8e4d01fa47bbb17471acf3cf0665 2024-11-15T16:59:54,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/2beea1f69d694f329de0748c7fb67ba3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/2beea1f69d694f329de0748c7fb67ba3 2024-11-15T16:59:54,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0104e54635be4787a4e76f724afc7bc5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/0104e54635be4787a4e76f724afc7bc5 2024-11-15T16:59:54,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/32415ca0067a4124911b4f3da38fd389 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/32415ca0067a4124911b4f3da38fd389 2024-11-15T16:59:54,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/f3ebb6eb1ba3470688833aae1de1cdd4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/f3ebb6eb1ba3470688833aae1de1cdd4 2024-11-15T16:59:54,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/39afc855fe7444b7b5585f7606e4adfb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/39afc855fe7444b7b5585f7606e4adfb 2024-11-15T16:59:54,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/641f5483fd3c4afa9ec578a88b512fd7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/641f5483fd3c4afa9ec578a88b512fd7 2024-11-15T16:59:54,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e83a42e8bac340f3b6e699f63ea696d4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/e83a42e8bac340f3b6e699f63ea696d4 2024-11-15T16:59:54,604 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/d39bca495bf84e8ba619b2a15a5f6448 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/d39bca495bf84e8ba619b2a15a5f6448 2024-11-15T16:59:54,605 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/20d868b068d14d6f9c428e09bd4f4702 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/20d868b068d14d6f9c428e09bd4f4702 2024-11-15T16:59:54,817 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/53f992ab2ebc4c2db522dc14be91786e 2024-11-15T16:59:54,823 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/ef28817937fe41d0938bbdff91b4c8c5 is 50, key is test_row_2/C:col10/1731689993426/Put/seqid=0 2024-11-15T16:59:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742239_1415 (size=7415) 2024-11-15T16:59:55,227 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/ef28817937fe41d0938bbdff91b4c8c5 2024-11-15T16:59:55,231 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/A/e128fdca340947a5b39bac02ccfa56f2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e128fdca340947a5b39bac02ccfa56f2 2024-11-15T16:59:55,234 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e128fdca340947a5b39bac02ccfa56f2, entries=50, sequenceid=335, filesize=13.5 K 2024-11-15T16:59:55,235 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/B/53f992ab2ebc4c2db522dc14be91786e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/53f992ab2ebc4c2db522dc14be91786e 2024-11-15T16:59:55,237 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/53f992ab2ebc4c2db522dc14be91786e, entries=50, sequenceid=335, filesize=7.2 K 2024-11-15T16:59:55,238 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/.tmp/C/ef28817937fe41d0938bbdff91b4c8c5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/ef28817937fe41d0938bbdff91b4c8c5 2024-11-15T16:59:55,240 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/ef28817937fe41d0938bbdff91b4c8c5, entries=50, sequenceid=335, filesize=7.2 K 2024-11-15T16:59:55,241 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fec8d5aa95a6ec0fca101912a0d79069 in 1652ms, sequenceid=335, compaction requested=true 2024-11-15T16:59:55,245 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/recovered.edits/338.seqid, newMaxSeqId=338, maxSeqId=4 2024-11-15T16:59:55,246 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. 2024-11-15T16:59:55,246 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for fec8d5aa95a6ec0fca101912a0d79069: Waiting for close lock at 1731689993589Running coprocessor pre-close hooks at 1731689993589Disabling compacts and flushes for region at 1731689993589Disabling writes for close at 1731689993589Obtaining lock to block concurrent updates at 1731689993590 (+1 ms)Preparing flush snapshotting stores in fec8d5aa95a6ec0fca101912a0d79069 at 1731689993590Finished memstore snapshotting TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731689993590Flushing stores of TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069. at 1731689993590Flushing fec8d5aa95a6ec0fca101912a0d79069/A: creating writer at 1731689993590Flushing mob file fec8d5aa95a6ec0fca101912a0d79069/A: appending metadata at 1731689993595 (+5 ms)Flushing mob file fec8d5aa95a6ec0fca101912a0d79069/A: closing flushed file at 1731689993595Flushing fec8d5aa95a6ec0fca101912a0d79069/A: appending metadata at 1731689994003 (+408 ms)Flushing fec8d5aa95a6ec0fca101912a0d79069/A: closing flushed file at 1731689994003Flushing fec8d5aa95a6ec0fca101912a0d79069/B: creating writer at 1731689994411 (+408 ms)Flushing fec8d5aa95a6ec0fca101912a0d79069/B: appending metadata at 1731689994413 (+2 ms)Flushing fec8d5aa95a6ec0fca101912a0d79069/B: closing flushed file at 1731689994413Flushing fec8d5aa95a6ec0fca101912a0d79069/C: creating writer at 1731689994821 (+408 ms)Flushing fec8d5aa95a6ec0fca101912a0d79069/C: appending metadata at 1731689994823 (+2 ms)Flushing fec8d5aa95a6ec0fca101912a0d79069/C: closing flushed file at 1731689994823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f521796: reopening flushed file at 1731689995231 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1281aad3: reopening flushed file at 1731689995234 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a078d5c: reopening flushed file at 1731689995237 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fec8d5aa95a6ec0fca101912a0d79069 in 1652ms, sequenceid=335, compaction requested=true at 1731689995241 (+4 ms)Writing region close event to WAL at 1731689995243 (+2 ms)Running coprocessor post-close hooks at 1731689995246 (+3 ms)Closed at 1731689995246 2024-11-15T16:59:55,247 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,248 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=fec8d5aa95a6ec0fca101912a0d79069, regionState=CLOSED 2024-11-15T16:59:55,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:59:55,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T16:59:55,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure fec8d5aa95a6ec0fca101912a0d79069, server=a9bde5b3a1b5,35119,1731689872090 in 1.8130 sec 2024-11-15T16:59:55,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-15T16:59:55,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fec8d5aa95a6ec0fca101912a0d79069, UNASSIGN in 1.8160 sec 2024-11-15T16:59:55,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T16:59:55,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8190 sec 2024-11-15T16:59:55,255 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689995255"}]},"ts":"1731689995255"} 2024-11-15T16:59:55,256 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T16:59:55,256 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T16:59:55,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8280 sec 2024-11-15T16:59:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T16:59:55,576 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:55,577 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T16:59:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:55,579 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T16:59:55,579 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:55,581 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,583 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/recovered.edits] 2024-11-15T16:59:55,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/8a3e98a24af74fe5b3b08dd92e455f8f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/8a3e98a24af74fe5b3b08dd92e455f8f 2024-11-15T16:59:55,586 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e128fdca340947a5b39bac02ccfa56f2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/e128fdca340947a5b39bac02ccfa56f2 2024-11-15T16:59:55,587 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/f2a79541d3b14d6282e762e7ac5dab26 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/A/f2a79541d3b14d6282e762e7ac5dab26 2024-11-15T16:59:55,589 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/53f992ab2ebc4c2db522dc14be91786e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/53f992ab2ebc4c2db522dc14be91786e 2024-11-15T16:59:55,590 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/a8955c498fb640058e102a69bed9cd58 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/a8955c498fb640058e102a69bed9cd58 2024-11-15T16:59:55,591 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/fedebda8fa244990939958135e126a0c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/B/fedebda8fa244990939958135e126a0c 2024-11-15T16:59:55,593 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/42630d3f8c684e158a411fae7cefd0f3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/42630d3f8c684e158a411fae7cefd0f3 2024-11-15T16:59:55,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/677f95cf87654d8c939dafda0418e3ec to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/677f95cf87654d8c939dafda0418e3ec 2024-11-15T16:59:55,595 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/ef28817937fe41d0938bbdff91b4c8c5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/C/ef28817937fe41d0938bbdff91b4c8c5 2024-11-15T16:59:55,597 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/recovered.edits/338.seqid to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069/recovered.edits/338.seqid 2024-11-15T16:59:55,597 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,597 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T16:59:55,598 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T16:59:55,598 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T16:59:55,600 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115048cb8cbc8b14923855792c97b0ae876_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115048cb8cbc8b14923855792c97b0ae876_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,601 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150b96faf4db3c41d28db9ed33cb8e8ede_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150b96faf4db3c41d28db9ed33cb8e8ede_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,602 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115138b449ee1b4433fb87435f98bce558d_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115138b449ee1b4433fb87435f98bce558d_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,603 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111548355313d24a4cfa93586887cccca16a_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111548355313d24a4cfa93586887cccca16a_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,603 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155d3aeb3097324874b5bc549859aa3ad0_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155d3aeb3097324874b5bc549859aa3ad0_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,604 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111561a313145c1f4b329e4d8e9cd643b778_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111561a313145c1f4b329e4d8e9cd643b778_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,605 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111564375233df9748a992510dc49a601ac4_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111564375233df9748a992510dc49a601ac4_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,606 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157259d464417e4c3ebf22f48c156f54e3_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157259d464417e4c3ebf22f48c156f54e3_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,607 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c0d3e411a7e4f8388492cc22a537426_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c0d3e411a7e4f8388492cc22a537426_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,608 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a9cc948ae7da4b6ba94370cbbf82dce8_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a9cc948ae7da4b6ba94370cbbf82dce8_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,609 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115af01570d9f3f4225a9d66c44e4b9e3ef_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115af01570d9f3f4225a9d66c44e4b9e3ef_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,610 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b562933e5184496f872c1a85d5e38829_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b562933e5184496f872c1a85d5e38829_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,610 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfa7d812289d4664bca585f040894899_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfa7d812289d4664bca585f040894899_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,611 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d36829bd9cd3437ebbac03a3e7297663_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d36829bd9cd3437ebbac03a3e7297663_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,612 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dcf81a291a1f472a852cef819c2c2296_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dcf81a291a1f472a852cef819c2c2296_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,613 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e89f0581914e46bdad73a5098a8ff88e_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e89f0581914e46bdad73a5098a8ff88e_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,615 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e969f5e01189450db22b6416e1b17459_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e969f5e01189450db22b6416e1b17459_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,616 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec167ed4556946dcab591aa774c86d7e_fec8d5aa95a6ec0fca101912a0d79069 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec167ed4556946dcab591aa774c86d7e_fec8d5aa95a6ec0fca101912a0d79069 2024-11-15T16:59:55,616 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T16:59:55,618 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:55,620 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T16:59:55,621 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T16:59:55,622 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:55,622 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T16:59:55,622 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731689995622"}]},"ts":"9223372036854775807"} 2024-11-15T16:59:55,624 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T16:59:55,624 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fec8d5aa95a6ec0fca101912a0d79069, NAME => 'TestAcidGuarantees,,1731689964576.fec8d5aa95a6ec0fca101912a0d79069.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T16:59:55,624 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T16:59:55,624 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731689995624"}]},"ts":"9223372036854775807"} 2024-11-15T16:59:55,626 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T16:59:55,626 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:55,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-15T16:59:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T16:59:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T16:59:55,686 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:55,697 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 238), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 311), ProcessCount=11 (was 11), AvailableMemoryMB=5288 (was 5426) 2024-11-15T16:59:55,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=5286 2024-11-15T16:59:55,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T16:59:55,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T16:59:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T16:59:55,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T16:59:55,711 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:55,711 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-15T16:59:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-15T16:59:55,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T16:59:55,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742240_1416 (size=959) 2024-11-15T16:59:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-15T16:59:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-15T16:59:56,118 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T16:59:56,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742241_1417 (size=53) 2024-11-15T16:59:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-15T16:59:56,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:59:56,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5304b51df6e8f78431676b542bf430ab, disabling compactions & flushes 2024-11-15T16:59:56,523 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:56,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:56,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. after waiting 1 ms 2024-11-15T16:59:56,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:56,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:56,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5304b51df6e8f78431676b542bf430ab: Waiting for close lock at 1731689996523Disabling compacts and flushes for region at 1731689996523Disabling writes for close at 1731689996524 (+1 ms)Writing region close event to WAL at 1731689996524Closed at 1731689996524 2024-11-15T16:59:56,525 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T16:59:56,525 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731689996525"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731689996525"}]},"ts":"1731689996525"} 2024-11-15T16:59:56,526 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T16:59:56,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T16:59:56,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689996527"}]},"ts":"1731689996527"} 2024-11-15T16:59:56,528 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T16:59:56,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, ASSIGN}] 2024-11-15T16:59:56,529 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, ASSIGN 2024-11-15T16:59:56,530 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=false 2024-11-15T16:59:56,680 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=5304b51df6e8f78431676b542bf430ab, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:56,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, ASSIGN because future has completed 2024-11-15T16:59:56,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T16:59:56,837 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:56,837 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} 2024-11-15T16:59:56,837 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,837 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T16:59:56,837 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,838 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,839 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,840 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5304b51df6e8f78431676b542bf430ab columnFamilyName A 2024-11-15T16:59:56,840 DEBUG [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:56,840 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] regionserver.HStore(327): Store=5304b51df6e8f78431676b542bf430ab/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:56,840 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,841 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5304b51df6e8f78431676b542bf430ab columnFamilyName B 2024-11-15T16:59:56,841 DEBUG [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:56,842 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] regionserver.HStore(327): Store=5304b51df6e8f78431676b542bf430ab/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:56,842 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,843 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5304b51df6e8f78431676b542bf430ab columnFamilyName C 2024-11-15T16:59:56,843 DEBUG [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T16:59:56,843 INFO [StoreOpener-5304b51df6e8f78431676b542bf430ab-1 {}] regionserver.HStore(327): Store=5304b51df6e8f78431676b542bf430ab/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T16:59:56,843 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,844 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,844 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,845 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,845 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,845 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T16:59:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-15T16:59:56,846 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,847 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T16:59:56,848 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 5304b51df6e8f78431676b542bf430ab; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75456131, jitterRate=0.12438397109508514}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T16:59:56,848 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:56,848 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 5304b51df6e8f78431676b542bf430ab: Running coprocessor pre-open hook at 1731689996838Writing region info on filesystem at 1731689996838Initializing all the Stores at 1731689996838Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689996838Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689996838Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731689996838Cleaning up temporary data from old regions at 1731689996845 (+7 ms)Running coprocessor post-open hooks at 1731689996848 (+3 ms)Region opened successfully at 1731689996848 2024-11-15T16:59:56,849 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., pid=124, masterSystemTime=1731689996834 2024-11-15T16:59:56,851 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:56,851 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:56,851 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=5304b51df6e8f78431676b542bf430ab, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:56,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T16:59:56,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T16:59:56,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 in 171 msec 2024-11-15T16:59:56,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-15T16:59:56,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, ASSIGN in 326 msec 2024-11-15T16:59:56,856 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T16:59:56,856 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731689996856"}]},"ts":"1731689996856"} 2024-11-15T16:59:56,857 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T16:59:56,858 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T16:59:56,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-15T16:59:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-15T16:59:57,856 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T16:59:57,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,858 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a90cc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,859 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77fa9c83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,860 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,863 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T16:59:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,41785,1731689871395 2024-11-15T16:59:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@166c1cc7 2024-11-15T16:59:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T16:59:57,864 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T16:59:57,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,866 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0c1284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,867 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ecd873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,868 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716f9696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62647c93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,871 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579292ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,872 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f473af2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,873 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53008179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a16c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44274587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363b02f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509d41af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a39a221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b701710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,882 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f2780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,883 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@437e80e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,884 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fba8af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,886 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59a13212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,887 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0396c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T16:59:57,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T16:59:57,888 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T16:59:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T16:59:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T16:59:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcc3641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T16:59:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T16:59:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T16:59:57,890 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T16:59:57,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708f7d85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T16:59:57,891 DEBUG [Thread-1894 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,891 DEBUG [Thread-1887 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,891 DEBUG [Thread-1885 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,891 DEBUG [Thread-1891 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,892 DEBUG [Thread-1898 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,893 DEBUG [Thread-1883 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,893 DEBUG [Thread-1900 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,894 DEBUG [Thread-1889 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,895 DEBUG [Thread-1896 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,898 DEBUG [Thread-1902 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T16:59:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T16:59:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T16:59:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T16:59:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2] 2024-11-15T16:59:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T16:59:57,904 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T16:59:57,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T16:59:57,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T16:59:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T16:59:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T16:59:57,912 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T16:59:57,922 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T16:59:57,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T16:59:57,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c038b334e2574753b329d8884d71e9cf is 50, key is test_row_0/A:col10/1731689997903/Put/seqid=0 2024-11-15T16:59:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742242_1418 (size=12001) 2024-11-15T16:59:57,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690057950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:57,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:57,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690057951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690057962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690057962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690057962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T16:59:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690058066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690058066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690058076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690058076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690058076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T16:59:58,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690058276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690058276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690058285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690058285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690058285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c038b334e2574753b329d8884d71e9cf 2024-11-15T16:59:58,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/f96606ae9a5f4b24a5d666ef1a070bf2 is 50, key is test_row_0/B:col10/1731689997903/Put/seqid=0 2024-11-15T16:59:58,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742243_1419 (size=12001) 2024-11-15T16:59:58,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:58,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:58,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T16:59:58,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690058586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690058586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690058595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690058595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690058595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:58,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:58,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:58,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/f96606ae9a5f4b24a5d666ef1a070bf2 2024-11-15T16:59:58,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/864b02a6dce04309ba8d84a5c2e78c5d is 50, key is test_row_0/C:col10/1731689997903/Put/seqid=0 2024-11-15T16:59:58,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742244_1420 (size=12001) 2024-11-15T16:59:58,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:58,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:58,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:58,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T16:59:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690059106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690059106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690059106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:59,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690059116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T16:59:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690059116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T16:59:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T16:59:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T16:59:59,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:59,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:59,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T16:59:59,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:59,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:59,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T16:59:59,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/864b02a6dce04309ba8d84a5c2e78c5d 2024-11-15T16:59:59,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c038b334e2574753b329d8884d71e9cf as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c038b334e2574753b329d8884d71e9cf 2024-11-15T16:59:59,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c038b334e2574753b329d8884d71e9cf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T16:59:59,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/f96606ae9a5f4b24a5d666ef1a070bf2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/f96606ae9a5f4b24a5d666ef1a070bf2 2024-11-15T16:59:59,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/f96606ae9a5f4b24a5d666ef1a070bf2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T16:59:59,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/864b02a6dce04309ba8d84a5c2e78c5d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/864b02a6dce04309ba8d84a5c2e78c5d 2024-11-15T16:59:59,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/864b02a6dce04309ba8d84a5c2e78c5d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T16:59:59,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 5304b51df6e8f78431676b542bf430ab in 1315ms, sequenceid=13, compaction requested=false 2024-11-15T16:59:59,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T16:59:59,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T16:59:59,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T16:59:59,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T16:59:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e83d60cf77a546dc9f764e9ab207b1ed is 50, key is test_row_0/A:col10/1731689997960/Put/seqid=0 2024-11-15T16:59:59,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742245_1421 (size=12001) 2024-11-15T16:59:59,333 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e83d60cf77a546dc9f764e9ab207b1ed 2024-11-15T16:59:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/edd5d322cc3049338ff6812941c53b6f is 50, key is test_row_0/B:col10/1731689997960/Put/seqid=0 2024-11-15T16:59:59,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742246_1422 (size=12001) 2024-11-15T16:59:59,747 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/edd5d322cc3049338ff6812941c53b6f 2024-11-15T16:59:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/83fad30f349644aba855957bc18a6ce7 is 50, key is test_row_0/C:col10/1731689997960/Put/seqid=0 2024-11-15T16:59:59,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742247_1423 (size=12001) 2024-11-15T17:00:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:00:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:00,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:00,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690060138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690060140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690060140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690060141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690060142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,158 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/83fad30f349644aba855957bc18a6ce7 2024-11-15T17:00:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e83d60cf77a546dc9f764e9ab207b1ed as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e83d60cf77a546dc9f764e9ab207b1ed 2024-11-15T17:00:00,167 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e83d60cf77a546dc9f764e9ab207b1ed, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T17:00:00,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/edd5d322cc3049338ff6812941c53b6f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/edd5d322cc3049338ff6812941c53b6f 2024-11-15T17:00:00,172 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/edd5d322cc3049338ff6812941c53b6f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T17:00:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/83fad30f349644aba855957bc18a6ce7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/83fad30f349644aba855957bc18a6ce7 2024-11-15T17:00:00,177 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/83fad30f349644aba855957bc18a6ce7, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T17:00:00,178 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5304b51df6e8f78431676b542bf430ab in 878ms, sequenceid=38, compaction requested=false 2024-11-15T17:00:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T17:00:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T17:00:00,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T17:00:00,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2580 sec 2024-11-15T17:00:00,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.2770 sec 2024-11-15T17:00:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:00,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:00,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:00:00,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c671c2b8c32b4d8282f1fd4bd5097466 is 50, key is test_row_0/A:col10/1731690000249/Put/seqid=0 2024-11-15T17:00:00,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742248_1424 (size=14341) 2024-11-15T17:00:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690060290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690060291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690060294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690060295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690060296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690060406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690060406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690060406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690060406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690060415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690060616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690060616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690060625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690060625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690060635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c671c2b8c32b4d8282f1fd4bd5097466 2024-11-15T17:00:00,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/998c8bbaf8474ffe9f10cb04cb88cfcb is 50, key is test_row_0/B:col10/1731690000249/Put/seqid=0 2024-11-15T17:00:00,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742249_1425 (size=12001) 2024-11-15T17:00:00,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/998c8bbaf8474ffe9f10cb04cb88cfcb 2024-11-15T17:00:00,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/d9ba4fca91534634b407aa5edc6bef39 is 50, key is test_row_0/C:col10/1731690000249/Put/seqid=0 2024-11-15T17:00:00,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742250_1426 (size=12001) 2024-11-15T17:00:00,766 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:00:00,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690060925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690060926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690060946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690060946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690060955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/d9ba4fca91534634b407aa5edc6bef39 2024-11-15T17:00:01,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c671c2b8c32b4d8282f1fd4bd5097466 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c671c2b8c32b4d8282f1fd4bd5097466 2024-11-15T17:00:01,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c671c2b8c32b4d8282f1fd4bd5097466, entries=200, sequenceid=50, filesize=14.0 K 2024-11-15T17:00:01,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/998c8bbaf8474ffe9f10cb04cb88cfcb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/998c8bbaf8474ffe9f10cb04cb88cfcb 2024-11-15T17:00:01,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/998c8bbaf8474ffe9f10cb04cb88cfcb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T17:00:01,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/d9ba4fca91534634b407aa5edc6bef39 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d9ba4fca91534634b407aa5edc6bef39 2024-11-15T17:00:01,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d9ba4fca91534634b407aa5edc6bef39, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T17:00:01,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5304b51df6e8f78431676b542bf430ab in 855ms, sequenceid=50, compaction requested=true 2024-11-15T17:00:01,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:01,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:01,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:01,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:01,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:01,106 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:01,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:01,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:01,106 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:01,107 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:01,107 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:01,107 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:01,107 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:01,107 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:01,107 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:01,108 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/f96606ae9a5f4b24a5d666ef1a070bf2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/edd5d322cc3049338ff6812941c53b6f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/998c8bbaf8474ffe9f10cb04cb88cfcb] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=35.2 K 2024-11-15T17:00:01,108 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c038b334e2574753b329d8884d71e9cf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e83d60cf77a546dc9f764e9ab207b1ed, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c671c2b8c32b4d8282f1fd4bd5097466] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=37.4 K 2024-11-15T17:00:01,108 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f96606ae9a5f4b24a5d666ef1a070bf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731689997897 2024-11-15T17:00:01,108 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c038b334e2574753b329d8884d71e9cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731689997897 2024-11-15T17:00:01,108 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83d60cf77a546dc9f764e9ab207b1ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689997945 2024-11-15T17:00:01,108 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting edd5d322cc3049338ff6812941c53b6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689997945 2024-11-15T17:00:01,109 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c671c2b8c32b4d8282f1fd4bd5097466, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731690000138 2024-11-15T17:00:01,109 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 998c8bbaf8474ffe9f10cb04cb88cfcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731690000138 2024-11-15T17:00:01,115 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:01,115 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:01,116 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/e78a7858ad6e4caea63f95fa38755622 is 50, key is test_row_0/B:col10/1731690000249/Put/seqid=0 2024-11-15T17:00:01,116 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/50d1188714f34ffba96d8b138ef52044 is 50, key is test_row_0/A:col10/1731690000249/Put/seqid=0 2024-11-15T17:00:01,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742251_1427 (size=12104) 2024-11-15T17:00:01,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742252_1428 (size=12104) 2024-11-15T17:00:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:01,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:00:01,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/25a5447462e648bdbae2e532ffa1c272 is 50, key is test_row_0/A:col10/1731690000281/Put/seqid=0 2024-11-15T17:00:01,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742253_1429 (size=14341) 2024-11-15T17:00:01,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690061465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690061466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690061467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690061468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690061472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,531 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/e78a7858ad6e4caea63f95fa38755622 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/e78a7858ad6e4caea63f95fa38755622 2024-11-15T17:00:01,536 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/50d1188714f34ffba96d8b138ef52044 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/50d1188714f34ffba96d8b138ef52044 2024-11-15T17:00:01,538 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into e78a7858ad6e4caea63f95fa38755622(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:01,538 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:01,538 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=13, startTime=1731690001106; duration=0sec 2024-11-15T17:00:01,538 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:01,538 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:01,538 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:01,539 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:01,539 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:01,539 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:01,539 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/864b02a6dce04309ba8d84a5c2e78c5d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/83fad30f349644aba855957bc18a6ce7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d9ba4fca91534634b407aa5edc6bef39] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=35.2 K 2024-11-15T17:00:01,540 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 864b02a6dce04309ba8d84a5c2e78c5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731689997897 2024-11-15T17:00:01,540 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 83fad30f349644aba855957bc18a6ce7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731689997945 2024-11-15T17:00:01,540 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d9ba4fca91534634b407aa5edc6bef39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731690000138 2024-11-15T17:00:01,541 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into 50d1188714f34ffba96d8b138ef52044(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:01,541 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:01,541 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=13, startTime=1731690001106; duration=0sec 2024-11-15T17:00:01,541 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:01,541 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:01,547 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:01,547 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/33d42da1337a49288adf1d675e62d22f is 50, key is test_row_0/C:col10/1731690000249/Put/seqid=0 2024-11-15T17:00:01,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742254_1430 (size=12104) 2024-11-15T17:00:01,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690061585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690061585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690061586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690061586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690061795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690061805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690061806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690061806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:01,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/25a5447462e648bdbae2e532ffa1c272 2024-11-15T17:00:01,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/6c77bc3eef4944099089b74205ede63f is 50, key is test_row_0/B:col10/1731690000281/Put/seqid=0 2024-11-15T17:00:01,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742255_1431 (size=12001) 2024-11-15T17:00:01,956 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/33d42da1337a49288adf1d675e62d22f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/33d42da1337a49288adf1d675e62d22f 2024-11-15T17:00:01,960 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into 33d42da1337a49288adf1d675e62d22f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:01,960 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:01,961 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=13, startTime=1731690001106; duration=0sec 2024-11-15T17:00:01,961 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:01,961 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:00:02,065 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:02,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T17:00:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:00:02,068 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:02,069 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:02,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:02,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690062116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690062126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690062126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690062126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:00:02,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:00:02,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:02,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/6c77bc3eef4944099089b74205ede63f 2024-11-15T17:00:02,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/a4f5949f9fc9428286bfa087352ec2fc is 50, key is test_row_0/C:col10/1731690000281/Put/seqid=0 2024-11-15T17:00:02,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742256_1432 (size=12001) 2024-11-15T17:00:02,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:00:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:02,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:00:02,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690062485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,489 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 5304b51df6e8f78431676b542bf430ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:00:02,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:02,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690062635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690062635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690062646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690062646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:02,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/a4f5949f9fc9428286bfa087352ec2fc 2024-11-15T17:00:02,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:00:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/25a5447462e648bdbae2e532ffa1c272 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/25a5447462e648bdbae2e532ffa1c272 2024-11-15T17:00:02,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/25a5447462e648bdbae2e532ffa1c272, entries=200, sequenceid=75, filesize=14.0 K 2024-11-15T17:00:02,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/6c77bc3eef4944099089b74205ede63f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/6c77bc3eef4944099089b74205ede63f 2024-11-15T17:00:02,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/6c77bc3eef4944099089b74205ede63f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T17:00:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/a4f5949f9fc9428286bfa087352ec2fc as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/a4f5949f9fc9428286bfa087352ec2fc 2024-11-15T17:00:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:00:02,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/a4f5949f9fc9428286bfa087352ec2fc, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T17:00:02,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5304b51df6e8f78431676b542bf430ab in 1261ms, sequenceid=75, compaction requested=false 2024-11-15T17:00:02,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:02,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:00:02,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:02,834 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:00:02,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/72893ee5aa0b4aa987c38f5014045daf is 50, key is test_row_0/A:col10/1731690001462/Put/seqid=0 2024-11-15T17:00:02,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742257_1433 (size=12001) 2024-11-15T17:00:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:00:03,243 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/72893ee5aa0b4aa987c38f5014045daf 2024-11-15T17:00:03,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/0af8e04c29e54b3780c7cf6370dccc84 is 50, key is test_row_0/B:col10/1731690001462/Put/seqid=0 2024-11-15T17:00:03,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742258_1434 (size=12001) 2024-11-15T17:00:03,658 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/0af8e04c29e54b3780c7cf6370dccc84 2024-11-15T17:00:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/abf13351a05942b8b7bd1337b938ea60 is 50, key is test_row_0/C:col10/1731690001462/Put/seqid=0 2024-11-15T17:00:03,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742259_1435 (size=12001) 2024-11-15T17:00:03,673 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/abf13351a05942b8b7bd1337b938ea60 2024-11-15T17:00:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/72893ee5aa0b4aa987c38f5014045daf as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/72893ee5aa0b4aa987c38f5014045daf 2024-11-15T17:00:03,682 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/72893ee5aa0b4aa987c38f5014045daf, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T17:00:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/0af8e04c29e54b3780c7cf6370dccc84 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/0af8e04c29e54b3780c7cf6370dccc84 2024-11-15T17:00:03,687 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/0af8e04c29e54b3780c7cf6370dccc84, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T17:00:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/abf13351a05942b8b7bd1337b938ea60 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/abf13351a05942b8b7bd1337b938ea60 2024-11-15T17:00:03,692 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/abf13351a05942b8b7bd1337b938ea60, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T17:00:03,693 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=80.51 KB/82440 for 5304b51df6e8f78431676b542bf430ab in 859ms, sequenceid=89, compaction requested=true 2024-11-15T17:00:03,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:03,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:03,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T17:00:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T17:00:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:03,696 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:03,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T17:00:03,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T17:00:03,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6250 sec 2024-11-15T17:00:03,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/fb9ab7a5451648ddb9369d658f854f7c is 50, key is test_row_0/A:col10/1731690003660/Put/seqid=0 2024-11-15T17:00:03,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6320 sec 2024-11-15T17:00:03,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742260_1436 (size=14341) 2024-11-15T17:00:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690063746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690063749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690063750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:03,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690063756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:03,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690063866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690063866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690063866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:03,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690063875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690064086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690064086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690064086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690064086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/fb9ab7a5451648ddb9369d658f854f7c 2024-11-15T17:00:04,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/23408a7b737445b18156592f76e1ce49 is 50, key is test_row_0/B:col10/1731690003660/Put/seqid=0 2024-11-15T17:00:04,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742261_1437 (size=12001) 2024-11-15T17:00:04,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/23408a7b737445b18156592f76e1ce49 2024-11-15T17:00:04,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/f60218f34fb84e049c6de5870fb74b81 is 50, key is test_row_0/C:col10/1731690003660/Put/seqid=0 2024-11-15T17:00:04,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742262_1438 (size=12001) 2024-11-15T17:00:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:00:04,216 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:04,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T17:00:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:00:04,218 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:04,218 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:04,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:00:04,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T17:00:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690064406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690064406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690064406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690064406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690064515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T17:00:04,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:04,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:04,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:04,525 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 5304b51df6e8f78431676b542bf430ab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:00:04,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/f60218f34fb84e049c6de5870fb74b81 2024-11-15T17:00:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/fb9ab7a5451648ddb9369d658f854f7c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/fb9ab7a5451648ddb9369d658f854f7c 2024-11-15T17:00:04,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/fb9ab7a5451648ddb9369d658f854f7c, entries=200, sequenceid=105, filesize=14.0 K 2024-11-15T17:00:04,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/23408a7b737445b18156592f76e1ce49 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/23408a7b737445b18156592f76e1ce49 2024-11-15T17:00:04,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/23408a7b737445b18156592f76e1ce49, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T17:00:04,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/f60218f34fb84e049c6de5870fb74b81 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f60218f34fb84e049c6de5870fb74b81 2024-11-15T17:00:04,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f60218f34fb84e049c6de5870fb74b81, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T17:00:04,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5304b51df6e8f78431676b542bf430ab in 866ms, sequenceid=105, compaction requested=true 2024-11-15T17:00:04,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:04,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:04,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:04,562 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:04,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:04,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:04,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:04,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:04,563 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:04,564 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:04,564 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:04,564 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,564 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/e78a7858ad6e4caea63f95fa38755622, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/6c77bc3eef4944099089b74205ede63f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/0af8e04c29e54b3780c7cf6370dccc84, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/23408a7b737445b18156592f76e1ce49] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=47.0 K 2024-11-15T17:00:04,564 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:04,564 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:04,565 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,565 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/50d1188714f34ffba96d8b138ef52044, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/25a5447462e648bdbae2e532ffa1c272, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/72893ee5aa0b4aa987c38f5014045daf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/fb9ab7a5451648ddb9369d658f854f7c] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=51.5 K 2024-11-15T17:00:04,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50d1188714f34ffba96d8b138ef52044, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731690000138 2024-11-15T17:00:04,565 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e78a7858ad6e4caea63f95fa38755622, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731690000138 2024-11-15T17:00:04,565 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a5447462e648bdbae2e532ffa1c272, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731690000281 2024-11-15T17:00:04,565 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c77bc3eef4944099089b74205ede63f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731690000281 2024-11-15T17:00:04,566 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72893ee5aa0b4aa987c38f5014045daf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731690001462 2024-11-15T17:00:04,566 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af8e04c29e54b3780c7cf6370dccc84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731690001462 2024-11-15T17:00:04,566 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb9ab7a5451648ddb9369d658f854f7c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731690003660 2024-11-15T17:00:04,566 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 23408a7b737445b18156592f76e1ce49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731690003660 2024-11-15T17:00:04,576 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:04,577 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/a8b0722758484a889ccd799945572fbc is 50, key is test_row_0/A:col10/1731690003660/Put/seqid=0 2024-11-15T17:00:04,580 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:04,580 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/cc9b6cc7d49c499d9e4c760ed1fc20a0 is 50, key is test_row_0/B:col10/1731690003660/Put/seqid=0 2024-11-15T17:00:04,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742263_1439 (size=12241) 2024-11-15T17:00:04,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742264_1440 (size=12241) 2024-11-15T17:00:04,603 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/cc9b6cc7d49c499d9e4c760ed1fc20a0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/cc9b6cc7d49c499d9e4c760ed1fc20a0 2024-11-15T17:00:04,611 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into cc9b6cc7d49c499d9e4c760ed1fc20a0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:04,611 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:04,611 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=12, startTime=1731690004562; duration=0sec 2024-11-15T17:00:04,611 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:04,611 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:04,611 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:04,612 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:04,612 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:04,612 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,612 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/33d42da1337a49288adf1d675e62d22f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/a4f5949f9fc9428286bfa087352ec2fc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/abf13351a05942b8b7bd1337b938ea60, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f60218f34fb84e049c6de5870fb74b81] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=47.0 K 2024-11-15T17:00:04,613 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d42da1337a49288adf1d675e62d22f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731690000138 2024-11-15T17:00:04,613 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f5949f9fc9428286bfa087352ec2fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731690000281 2024-11-15T17:00:04,613 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting abf13351a05942b8b7bd1337b938ea60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731690001462 2024-11-15T17:00:04,614 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f60218f34fb84e049c6de5870fb74b81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731690003660 2024-11-15T17:00:04,621 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:04,622 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/1abedcb6223d4fe7a1f070704bf2ae58 is 50, key is test_row_0/C:col10/1731690003660/Put/seqid=0 2024-11-15T17:00:04,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742265_1441 (size=12241) 2024-11-15T17:00:04,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T17:00:04,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:04,678 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:00:04,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/45873a20e0bb435db023a1eb221db9e7 is 50, key is test_row_0/A:col10/1731690003735/Put/seqid=0 2024-11-15T17:00:04,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742266_1442 (size=12001) 2024-11-15T17:00:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:00:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690064949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690064959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690064960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690064961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:04,990 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/a8b0722758484a889ccd799945572fbc as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a8b0722758484a889ccd799945572fbc 2024-11-15T17:00:04,994 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into a8b0722758484a889ccd799945572fbc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:04,994 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:04,994 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=12, startTime=1731690004562; duration=0sec 2024-11-15T17:00:04,994 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:04,994 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:05,030 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/1abedcb6223d4fe7a1f070704bf2ae58 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/1abedcb6223d4fe7a1f070704bf2ae58 2024-11-15T17:00:05,034 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into 1abedcb6223d4fe7a1f070704bf2ae58(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:05,034 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:05,034 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=12, startTime=1731690004562; duration=0sec 2024-11-15T17:00:05,035 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:05,035 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690065065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690065065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690065066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690065076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,088 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/45873a20e0bb435db023a1eb221db9e7 2024-11-15T17:00:05,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/7f169809a95f439099a4b579f90206e3 is 50, key is test_row_0/B:col10/1731690003735/Put/seqid=0 2024-11-15T17:00:05,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742267_1443 (size=12001) 2024-11-15T17:00:05,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690065286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690065286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690065286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690065296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:00:05,507 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/7f169809a95f439099a4b579f90206e3 2024-11-15T17:00:05,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/3bd89f22362e43a69a8348cc98f71e76 is 50, key is test_row_0/C:col10/1731690003735/Put/seqid=0 2024-11-15T17:00:05,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742268_1444 (size=12001) 2024-11-15T17:00:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690065595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690065596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690065596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690065606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:05,927 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/3bd89f22362e43a69a8348cc98f71e76 2024-11-15T17:00:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/45873a20e0bb435db023a1eb221db9e7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/45873a20e0bb435db023a1eb221db9e7 2024-11-15T17:00:05,935 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/45873a20e0bb435db023a1eb221db9e7, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T17:00:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/7f169809a95f439099a4b579f90206e3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/7f169809a95f439099a4b579f90206e3 2024-11-15T17:00:05,938 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/7f169809a95f439099a4b579f90206e3, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T17:00:05,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/3bd89f22362e43a69a8348cc98f71e76 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3bd89f22362e43a69a8348cc98f71e76 2024-11-15T17:00:05,942 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3bd89f22362e43a69a8348cc98f71e76, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T17:00:05,943 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5304b51df6e8f78431676b542bf430ab in 1265ms, sequenceid=126, compaction requested=false 2024-11-15T17:00:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T17:00:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T17:00:05,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T17:00:05,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7270 sec 2024-11-15T17:00:05,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7310 sec 2024-11-15T17:00:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:06,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:06,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:00:06,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/90f652ec78cc4d66b2f7a5cf109e3a3d is 50, key is test_row_0/A:col10/1731690006121/Put/seqid=0 2024-11-15T17:00:06,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742269_1445 (size=16931) 2024-11-15T17:00:06,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/90f652ec78cc4d66b2f7a5cf109e3a3d 2024-11-15T17:00:06,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/42d66ce8f98e4a13bacfdc20205b6877 is 50, key is test_row_0/B:col10/1731690006121/Put/seqid=0 2024-11-15T17:00:06,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742270_1446 (size=12151) 2024-11-15T17:00:06,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690066160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690066160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690066169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690066171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690066276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690066276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690066285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690066285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:00:06,367 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:06,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T17:00:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:00:06,370 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:06,370 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:06,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:00:06,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690066486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690066486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690066505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690066505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:00:06,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:06,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/42d66ce8f98e4a13bacfdc20205b6877 2024-11-15T17:00:06,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/5c9715853cee405d8d8744e2127ba6ea is 50, key is test_row_0/C:col10/1731690006121/Put/seqid=0 2024-11-15T17:00:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742271_1447 (size=12151) 2024-11-15T17:00:06,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:00:06,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:06,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:00:06,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690066796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690066796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690066815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690066815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:06,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:00:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:06,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/5c9715853cee405d8d8744e2127ba6ea 2024-11-15T17:00:06,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/90f652ec78cc4d66b2f7a5cf109e3a3d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/90f652ec78cc4d66b2f7a5cf109e3a3d 2024-11-15T17:00:06,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/90f652ec78cc4d66b2f7a5cf109e3a3d, entries=250, sequenceid=147, filesize=16.5 K 2024-11-15T17:00:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/42d66ce8f98e4a13bacfdc20205b6877 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/42d66ce8f98e4a13bacfdc20205b6877 2024-11-15T17:00:06,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/42d66ce8f98e4a13bacfdc20205b6877, entries=150, sequenceid=147, filesize=11.9 K 2024-11-15T17:00:06,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/5c9715853cee405d8d8744e2127ba6ea as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/5c9715853cee405d8d8744e2127ba6ea 2024-11-15T17:00:06,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/5c9715853cee405d8d8744e2127ba6ea, entries=150, sequenceid=147, filesize=11.9 K 2024-11-15T17:00:06,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5304b51df6e8f78431676b542bf430ab in 854ms, sequenceid=147, compaction requested=true 2024-11-15T17:00:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:06,976 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:06,976 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:06,977 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:06,977 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:06,977 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,977 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a8b0722758484a889ccd799945572fbc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/45873a20e0bb435db023a1eb221db9e7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/90f652ec78cc4d66b2f7a5cf109e3a3d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=40.2 K 2024-11-15T17:00:06,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:06,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:06,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:06,977 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b0722758484a889ccd799945572fbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731690003660 2024-11-15T17:00:06,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:06,978 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45873a20e0bb435db023a1eb221db9e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731690003731 2024-11-15T17:00:06,978 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:06,978 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:06,978 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,978 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/cc9b6cc7d49c499d9e4c760ed1fc20a0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/7f169809a95f439099a4b579f90206e3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/42d66ce8f98e4a13bacfdc20205b6877] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=35.5 K 2024-11-15T17:00:06,979 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f652ec78cc4d66b2f7a5cf109e3a3d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731690004942 2024-11-15T17:00:06,979 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9b6cc7d49c499d9e4c760ed1fc20a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731690003660 2024-11-15T17:00:06,979 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f169809a95f439099a4b579f90206e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731690003731 2024-11-15T17:00:06,979 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d66ce8f98e4a13bacfdc20205b6877, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731690004959 2024-11-15T17:00:06,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:00:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:06,985 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T17:00:06,987 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:06,987 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e4aa699cf7694c55bd84bf359fa4de41 is 50, key is test_row_0/A:col10/1731690006121/Put/seqid=0 2024-11-15T17:00:06,990 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:06,991 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/d731731b95d14bc485f927202f072a69 is 50, key is test_row_0/B:col10/1731690006121/Put/seqid=0 2024-11-15T17:00:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/d98eb169154a4ff4abefaf605a962aaf is 50, key is test_row_0/A:col10/1731690006167/Put/seqid=0 2024-11-15T17:00:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:00:07,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742273_1449 (size=12493) 2024-11-15T17:00:07,013 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/d731731b95d14bc485f927202f072a69 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/d731731b95d14bc485f927202f072a69 2024-11-15T17:00:07,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742272_1448 (size=12493) 2024-11-15T17:00:07,022 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into d731731b95d14bc485f927202f072a69(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:07,022 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:07,022 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=13, startTime=1731690006976; duration=0sec 2024-11-15T17:00:07,022 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:07,022 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:07,022 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:07,024 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:07,024 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:07,024 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:07,024 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/1abedcb6223d4fe7a1f070704bf2ae58, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3bd89f22362e43a69a8348cc98f71e76, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/5c9715853cee405d8d8744e2127ba6ea] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=35.5 K 2024-11-15T17:00:07,024 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abedcb6223d4fe7a1f070704bf2ae58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731690003660 2024-11-15T17:00:07,025 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd89f22362e43a69a8348cc98f71e76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731690003731 2024-11-15T17:00:07,026 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c9715853cee405d8d8744e2127ba6ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731690004959 2024-11-15T17:00:07,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742274_1450 (size=12151) 2024-11-15T17:00:07,028 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e4aa699cf7694c55bd84bf359fa4de41 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e4aa699cf7694c55bd84bf359fa4de41 2024-11-15T17:00:07,028 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/d98eb169154a4ff4abefaf605a962aaf 2024-11-15T17:00:07,034 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into e4aa699cf7694c55bd84bf359fa4de41(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:07,034 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:07,034 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=13, startTime=1731690006976; duration=0sec 2024-11-15T17:00:07,034 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:07,034 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b2519d891bd64948b97bdf6c8d8109f4 is 50, key is test_row_0/B:col10/1731690006167/Put/seqid=0 2024-11-15T17:00:07,039 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:07,040 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/dbecbb9982ae48e0b6667ef2685b77fa is 50, key is test_row_0/C:col10/1731690006121/Put/seqid=0 2024-11-15T17:00:07,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742275_1451 (size=12151) 2024-11-15T17:00:07,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742276_1452 (size=12493) 2024-11-15T17:00:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:07,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:07,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690067346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690067347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690067348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690067349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,441 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b2519d891bd64948b97bdf6c8d8109f4 2024-11-15T17:00:07,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/586abdf12a26482c94e44f6298dc1b59 is 50, key is test_row_0/C:col10/1731690006167/Put/seqid=0 2024-11-15T17:00:07,455 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/dbecbb9982ae48e0b6667ef2685b77fa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dbecbb9982ae48e0b6667ef2685b77fa 2024-11-15T17:00:07,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742277_1453 (size=12151) 2024-11-15T17:00:07,457 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/586abdf12a26482c94e44f6298dc1b59 2024-11-15T17:00:07,461 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into dbecbb9982ae48e0b6667ef2685b77fa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:07,461 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:07,461 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=13, startTime=1731690006977; duration=0sec 2024-11-15T17:00:07,461 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:07,461 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:07,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/d98eb169154a4ff4abefaf605a962aaf as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/d98eb169154a4ff4abefaf605a962aaf 2024-11-15T17:00:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690067456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/d98eb169154a4ff4abefaf605a962aaf, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:00:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b2519d891bd64948b97bdf6c8d8109f4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b2519d891bd64948b97bdf6c8d8109f4 2024-11-15T17:00:07,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690067466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,472 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b2519d891bd64948b97bdf6c8d8109f4, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:00:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690067466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690067466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/586abdf12a26482c94e44f6298dc1b59 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/586abdf12a26482c94e44f6298dc1b59 2024-11-15T17:00:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/586abdf12a26482c94e44f6298dc1b59, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:00:07,478 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5304b51df6e8f78431676b542bf430ab in 493ms, sequenceid=165, compaction requested=false 2024-11-15T17:00:07,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:07,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:07,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T17:00:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T17:00:07,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T17:00:07,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1090 sec 2024-11-15T17:00:07,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.1150 sec 2024-11-15T17:00:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:00:07,506 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:07,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-15T17:00:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:00:07,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:07,509 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:07,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:00:07,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T17:00:07,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:07,663 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T17:00:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e17d08a607b8495f98859faff2a90e53 is 50, key is test_row_0/A:col10/1731690007346/Put/seqid=0 2024-11-15T17:00:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:07,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:07,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742278_1454 (size=12151) 2024-11-15T17:00:07,681 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e17d08a607b8495f98859faff2a90e53 2024-11-15T17:00:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/10cb9cd795284838b3ab466b6ebbdf4c is 50, key is test_row_0/B:col10/1731690007346/Put/seqid=0 2024-11-15T17:00:07,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742279_1455 (size=12151) 2024-11-15T17:00:07,698 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/10cb9cd795284838b3ab466b6ebbdf4c 2024-11-15T17:00:07,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/04154c3f22ed49c9b737cb27c9bec7e8 is 50, key is test_row_0/C:col10/1731690007346/Put/seqid=0 2024-11-15T17:00:07,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742280_1456 (size=12151) 2024-11-15T17:00:07,714 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/04154c3f22ed49c9b737cb27c9bec7e8 2024-11-15T17:00:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e17d08a607b8495f98859faff2a90e53 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e17d08a607b8495f98859faff2a90e53 2024-11-15T17:00:07,723 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e17d08a607b8495f98859faff2a90e53, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T17:00:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/10cb9cd795284838b3ab466b6ebbdf4c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/10cb9cd795284838b3ab466b6ebbdf4c 2024-11-15T17:00:07,729 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/10cb9cd795284838b3ab466b6ebbdf4c, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T17:00:07,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/04154c3f22ed49c9b737cb27c9bec7e8 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/04154c3f22ed49c9b737cb27c9bec7e8 2024-11-15T17:00:07,733 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/04154c3f22ed49c9b737cb27c9bec7e8, entries=150, sequenceid=186, filesize=11.9 K 2024-11-15T17:00:07,734 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 5304b51df6e8f78431676b542bf430ab in 72ms, sequenceid=186, compaction requested=true 2024-11-15T17:00:07,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:07,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:07,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-15T17:00:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-15T17:00:07,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T17:00:07,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 226 msec 2024-11-15T17:00:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:07,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:07,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T17:00:07,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 231 msec 2024-11-15T17:00:07,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c5c692de3c3248a6a1cdd1f973a5efa0 is 50, key is test_row_0/A:col10/1731690007704/Put/seqid=0 2024-11-15T17:00:07,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742281_1457 (size=14541) 2024-11-15T17:00:07,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c5c692de3c3248a6a1cdd1f973a5efa0 2024-11-15T17:00:07,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/218adbb2b1664ab981c09fff04fee175 is 50, key is test_row_0/B:col10/1731690007704/Put/seqid=0 2024-11-15T17:00:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690067766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690067769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742282_1458 (size=12151) 2024-11-15T17:00:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690067772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690067775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:00:07,826 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:07,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-15T17:00:07,828 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:00:07,828 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:07,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690067886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690067886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690067895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690067895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:00:07,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:00:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:07,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690068095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690068095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690068106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690068106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:00:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:00:08,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/218adbb2b1664ab981c09fff04fee175 2024-11-15T17:00:08,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/dccae92ed671465fb444f058766c4043 is 50, key is test_row_0/C:col10/1731690007704/Put/seqid=0 2024-11-15T17:00:08,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742283_1459 (size=12151) 2024-11-15T17:00:08,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:00:08,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:08,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690068405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690068405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690068416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690068426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:00:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:00:08,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690068565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,573 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 5304b51df6e8f78431676b542bf430ab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:00:08,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:08,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/dccae92ed671465fb444f058766c4043 2024-11-15T17:00:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c5c692de3c3248a6a1cdd1f973a5efa0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c5c692de3c3248a6a1cdd1f973a5efa0 2024-11-15T17:00:08,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c5c692de3c3248a6a1cdd1f973a5efa0, entries=200, sequenceid=208, filesize=14.2 K 2024-11-15T17:00:08,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/218adbb2b1664ab981c09fff04fee175 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/218adbb2b1664ab981c09fff04fee175 2024-11-15T17:00:08,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/218adbb2b1664ab981c09fff04fee175, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T17:00:08,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/dccae92ed671465fb444f058766c4043 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dccae92ed671465fb444f058766c4043 2024-11-15T17:00:08,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dccae92ed671465fb444f058766c4043, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T17:00:08,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5304b51df6e8f78431676b542bf430ab in 871ms, sequenceid=208, compaction requested=true 2024-11-15T17:00:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:08,611 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:08,611 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:08,612 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:08,612 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:08,612 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:08,612 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:08,613 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,613 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,613 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/d731731b95d14bc485f927202f072a69, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b2519d891bd64948b97bdf6c8d8109f4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/10cb9cd795284838b3ab466b6ebbdf4c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/218adbb2b1664ab981c09fff04fee175] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=47.8 K 2024-11-15T17:00:08,613 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e4aa699cf7694c55bd84bf359fa4de41, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/d98eb169154a4ff4abefaf605a962aaf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e17d08a607b8495f98859faff2a90e53, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c5c692de3c3248a6a1cdd1f973a5efa0] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=50.1 K 2024-11-15T17:00:08,613 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d731731b95d14bc485f927202f072a69, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731690004959 2024-11-15T17:00:08,613 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4aa699cf7694c55bd84bf359fa4de41, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731690004959 2024-11-15T17:00:08,613 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting b2519d891bd64948b97bdf6c8d8109f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731690006158 2024-11-15T17:00:08,613 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d98eb169154a4ff4abefaf605a962aaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731690006158 2024-11-15T17:00:08,614 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 10cb9cd795284838b3ab466b6ebbdf4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731690007345 2024-11-15T17:00:08,614 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e17d08a607b8495f98859faff2a90e53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731690007345 2024-11-15T17:00:08,614 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 218adbb2b1664ab981c09fff04fee175, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731690007704 2024-11-15T17:00:08,614 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c692de3c3248a6a1cdd1f973a5efa0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731690007704 2024-11-15T17:00:08,624 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:08,624 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/a33852f195524dfc83465384b1205c52 is 50, key is test_row_0/A:col10/1731690007704/Put/seqid=0 2024-11-15T17:00:08,628 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:08,628 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/1aedd2fdbed14e2db3fc2e154a2fec25 is 50, key is test_row_0/B:col10/1731690007704/Put/seqid=0 2024-11-15T17:00:08,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742284_1460 (size=12629) 2024-11-15T17:00:08,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742285_1461 (size=12629) 2024-11-15T17:00:08,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:00:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:08,752 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:00:08,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/16125b9ec7fd4f5caf4ebac6044b1972 is 50, key is test_row_0/A:col10/1731690007768/Put/seqid=0 2024-11-15T17:00:08,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742286_1462 (size=12151) 2024-11-15T17:00:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:00:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690068972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690068984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690068986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690068987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,045 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/a33852f195524dfc83465384b1205c52 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a33852f195524dfc83465384b1205c52 2024-11-15T17:00:09,045 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/1aedd2fdbed14e2db3fc2e154a2fec25 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/1aedd2fdbed14e2db3fc2e154a2fec25 2024-11-15T17:00:09,053 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into 1aedd2fdbed14e2db3fc2e154a2fec25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:09,053 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:09,053 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=12, startTime=1731690008611; duration=0sec 2024-11-15T17:00:09,053 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:09,053 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:09,053 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:09,057 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into a33852f195524dfc83465384b1205c52(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:09,057 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:09,057 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=12, startTime=1731690008611; duration=0sec 2024-11-15T17:00:09,057 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:09,057 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:09,058 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:09,058 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:09,058 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:09,058 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dbecbb9982ae48e0b6667ef2685b77fa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/586abdf12a26482c94e44f6298dc1b59, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/04154c3f22ed49c9b737cb27c9bec7e8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dccae92ed671465fb444f058766c4043] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=47.8 K 2024-11-15T17:00:09,058 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting dbecbb9982ae48e0b6667ef2685b77fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731690004959 2024-11-15T17:00:09,058 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 586abdf12a26482c94e44f6298dc1b59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731690006158 2024-11-15T17:00:09,059 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 04154c3f22ed49c9b737cb27c9bec7e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731690007345 2024-11-15T17:00:09,059 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting dccae92ed671465fb444f058766c4043, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731690007704 2024-11-15T17:00:09,067 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:09,067 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/ca652bd0b67b49c796b88a87fbd56ca3 is 50, key is test_row_0/C:col10/1731690007704/Put/seqid=0 2024-11-15T17:00:09,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742287_1463 (size=12629) 2024-11-15T17:00:09,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690069095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690069105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690069105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690069106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,164 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/16125b9ec7fd4f5caf4ebac6044b1972 2024-11-15T17:00:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/c53f98cfccc44bd585ed386f2d296847 is 50, key is test_row_0/B:col10/1731690007768/Put/seqid=0 2024-11-15T17:00:09,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742288_1464 (size=12151) 2024-11-15T17:00:09,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690069305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690069325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690069326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690069326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,483 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/ca652bd0b67b49c796b88a87fbd56ca3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/ca652bd0b67b49c796b88a87fbd56ca3 2024-11-15T17:00:09,488 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into ca652bd0b67b49c796b88a87fbd56ca3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:09,488 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:09,488 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=12, startTime=1731690008612; duration=0sec 2024-11-15T17:00:09,488 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:09,488 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:09,575 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/c53f98cfccc44bd585ed386f2d296847 2024-11-15T17:00:09,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/3d1e3d7de00f4b4090267e0dddd92c1a is 50, key is test_row_0/C:col10/1731690007768/Put/seqid=0 2024-11-15T17:00:09,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742289_1465 (size=12151) 2024-11-15T17:00:09,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690069616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690069645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690069645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690069646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:00:09,988 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/3d1e3d7de00f4b4090267e0dddd92c1a 2024-11-15T17:00:09,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/16125b9ec7fd4f5caf4ebac6044b1972 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/16125b9ec7fd4f5caf4ebac6044b1972 2024-11-15T17:00:09,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/16125b9ec7fd4f5caf4ebac6044b1972, entries=150, sequenceid=222, filesize=11.9 K 2024-11-15T17:00:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/c53f98cfccc44bd585ed386f2d296847 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c53f98cfccc44bd585ed386f2d296847 2024-11-15T17:00:10,001 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c53f98cfccc44bd585ed386f2d296847, entries=150, sequenceid=222, filesize=11.9 K 2024-11-15T17:00:10,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/3d1e3d7de00f4b4090267e0dddd92c1a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3d1e3d7de00f4b4090267e0dddd92c1a 2024-11-15T17:00:10,004 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3d1e3d7de00f4b4090267e0dddd92c1a, entries=150, sequenceid=222, filesize=11.9 K 2024-11-15T17:00:10,005 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5304b51df6e8f78431676b542bf430ab in 1253ms, sequenceid=222, compaction requested=false 2024-11-15T17:00:10,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:10,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:10,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-15T17:00:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-15T17:00:10,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T17:00:10,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-15T17:00:10,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.1830 sec 2024-11-15T17:00:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:10,130 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:10,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:00:10,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e5f5b2bfc7f6409f9abbee2887aec199 is 50, key is test_row_0/A:col10/1731690010128/Put/seqid=0 2024-11-15T17:00:10,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742290_1466 (size=14541) 2024-11-15T17:00:10,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690070175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690070176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690070178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690070178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690070296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690070296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690070296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690070296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690070506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690070506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690070506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690070506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e5f5b2bfc7f6409f9abbee2887aec199 2024-11-15T17:00:10,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/9c9221df69ac4c33a7f82364078649ef is 50, key is test_row_0/B:col10/1731690010128/Put/seqid=0 2024-11-15T17:00:10,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742291_1467 (size=12151) 2024-11-15T17:00:10,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690070815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690070815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690070815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690070816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:10,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/9c9221df69ac4c33a7f82364078649ef 2024-11-15T17:00:10,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/12505430417948e898c51540f3ecd4e2 is 50, key is test_row_0/C:col10/1731690010128/Put/seqid=0 2024-11-15T17:00:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742292_1468 (size=12151) 2024-11-15T17:00:11,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690071336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690071336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:11,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690071336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:11,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690071336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:11,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/12505430417948e898c51540f3ecd4e2 2024-11-15T17:00:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e5f5b2bfc7f6409f9abbee2887aec199 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e5f5b2bfc7f6409f9abbee2887aec199 2024-11-15T17:00:11,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e5f5b2bfc7f6409f9abbee2887aec199, entries=200, sequenceid=248, filesize=14.2 K 2024-11-15T17:00:11,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/9c9221df69ac4c33a7f82364078649ef as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/9c9221df69ac4c33a7f82364078649ef 2024-11-15T17:00:11,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/9c9221df69ac4c33a7f82364078649ef, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T17:00:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/12505430417948e898c51540f3ecd4e2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/12505430417948e898c51540f3ecd4e2 2024-11-15T17:00:11,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/12505430417948e898c51540f3ecd4e2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T17:00:11,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 5304b51df6e8f78431676b542bf430ab in 1246ms, sequenceid=248, compaction requested=true 2024-11-15T17:00:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:11,376 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:11,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:11,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:11,377 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:11,377 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:11,377 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:11,377 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:11,377 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:11,377 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:11,377 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:11,377 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/1aedd2fdbed14e2db3fc2e154a2fec25, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c53f98cfccc44bd585ed386f2d296847, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/9c9221df69ac4c33a7f82364078649ef] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.1 K 2024-11-15T17:00:11,378 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a33852f195524dfc83465384b1205c52, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/16125b9ec7fd4f5caf4ebac6044b1972, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e5f5b2bfc7f6409f9abbee2887aec199] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=38.4 K 2024-11-15T17:00:11,378 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aedd2fdbed14e2db3fc2e154a2fec25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731690007704 2024-11-15T17:00:11,378 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a33852f195524dfc83465384b1205c52, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731690007704 2024-11-15T17:00:11,379 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting c53f98cfccc44bd585ed386f2d296847, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731690007768 2024-11-15T17:00:11,379 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16125b9ec7fd4f5caf4ebac6044b1972, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731690007768 2024-11-15T17:00:11,379 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c9221df69ac4c33a7f82364078649ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731690008971 2024-11-15T17:00:11,379 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f5b2bfc7f6409f9abbee2887aec199, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731690008967 2024-11-15T17:00:11,386 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:11,386 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/90576d8c1b5245218c015f61b20d347f is 50, key is test_row_0/B:col10/1731690010128/Put/seqid=0 2024-11-15T17:00:11,389 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:11,390 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/9dbe5c231be643fcbe4dbb4e7cdceddc is 50, key is test_row_0/A:col10/1731690010128/Put/seqid=0 2024-11-15T17:00:11,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742293_1469 (size=12731) 2024-11-15T17:00:11,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742294_1470 (size=12731) 2024-11-15T17:00:11,404 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/9dbe5c231be643fcbe4dbb4e7cdceddc as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/9dbe5c231be643fcbe4dbb4e7cdceddc 2024-11-15T17:00:11,409 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into 9dbe5c231be643fcbe4dbb4e7cdceddc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:11,409 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:11,409 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=13, startTime=1731690011376; duration=0sec 2024-11-15T17:00:11,409 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:11,409 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:11,409 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:11,410 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:11,410 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:11,410 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:11,410 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/ca652bd0b67b49c796b88a87fbd56ca3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3d1e3d7de00f4b4090267e0dddd92c1a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/12505430417948e898c51540f3ecd4e2] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.1 K 2024-11-15T17:00:11,410 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca652bd0b67b49c796b88a87fbd56ca3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731690007704 2024-11-15T17:00:11,411 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1e3d7de00f4b4090267e0dddd92c1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731690007768 2024-11-15T17:00:11,411 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12505430417948e898c51540f3ecd4e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731690008971 2024-11-15T17:00:11,420 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:11,421 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/b7c3acedd1aa478993fa621250138579 is 50, key is test_row_0/C:col10/1731690010128/Put/seqid=0 2024-11-15T17:00:11,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742295_1471 (size=12731) 2024-11-15T17:00:11,796 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/90576d8c1b5245218c015f61b20d347f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/90576d8c1b5245218c015f61b20d347f 2024-11-15T17:00:11,800 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into 90576d8c1b5245218c015f61b20d347f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:11,800 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:11,800 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=13, startTime=1731690011376; duration=0sec 2024-11-15T17:00:11,800 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:11,800 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:11,828 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/b7c3acedd1aa478993fa621250138579 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/b7c3acedd1aa478993fa621250138579 2024-11-15T17:00:11,832 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into b7c3acedd1aa478993fa621250138579(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:11,832 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:11,832 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=13, startTime=1731690011376; duration=0sec 2024-11-15T17:00:11,832 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:11,832 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:00:11,996 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:11,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-15T17:00:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:00:11,998 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:11,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:11,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:00:12,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:00:12,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,151 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:00:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/4a1ec31657d94d44ba520e41a893c909 is 50, key is test_row_0/A:col10/1731690010176/Put/seqid=0 2024-11-15T17:00:12,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742296_1472 (size=12251) 2024-11-15T17:00:12,161 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/4a1ec31657d94d44ba520e41a893c909 2024-11-15T17:00:12,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/c58f646218914cbd8772c6471c6252f3 is 50, key is test_row_0/B:col10/1731690010176/Put/seqid=0 2024-11-15T17:00:12,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742297_1473 (size=12251) 2024-11-15T17:00:12,173 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/c58f646218914cbd8772c6471c6252f3 2024-11-15T17:00:12,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/afa78d78d4d14ff79d97836f32031fe5 is 50, key is test_row_0/C:col10/1731690010176/Put/seqid=0 2024-11-15T17:00:12,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742298_1474 (size=12251) 2024-11-15T17:00:12,182 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/afa78d78d4d14ff79d97836f32031fe5 2024-11-15T17:00:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/4a1ec31657d94d44ba520e41a893c909 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/4a1ec31657d94d44ba520e41a893c909 2024-11-15T17:00:12,189 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/4a1ec31657d94d44ba520e41a893c909, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T17:00:12,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/c58f646218914cbd8772c6471c6252f3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c58f646218914cbd8772c6471c6252f3 2024-11-15T17:00:12,193 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c58f646218914cbd8772c6471c6252f3, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T17:00:12,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/afa78d78d4d14ff79d97836f32031fe5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/afa78d78d4d14ff79d97836f32031fe5 2024-11-15T17:00:12,197 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/afa78d78d4d14ff79d97836f32031fe5, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T17:00:12,198 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 5304b51df6e8f78431676b542bf430ab in 47ms, sequenceid=265, compaction requested=false 2024-11-15T17:00:12,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:12,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-15T17:00:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-15T17:00:12,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T17:00:12,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 202 msec 2024-11-15T17:00:12,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 209 msec 2024-11-15T17:00:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:00:12,316 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:12,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-15T17:00:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:00:12,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:12,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:12,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:12,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:00:12,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/f692589e09ed467b9e60b16018b5bb79 is 50, key is test_row_0/A:col10/1731690012356/Put/seqid=0 2024-11-15T17:00:12,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742299_1475 (size=14741) 2024-11-15T17:00:12,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690072413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:00:12,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690072417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690072418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690072419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690072535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690072535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690072535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690072535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:12,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:00:12,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690072745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690072745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690072745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690072745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:12,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/f692589e09ed467b9e60b16018b5bb79 2024-11-15T17:00:12,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:12,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/8a7cbe32050e487ebb9f32431f4b291d is 50, key is test_row_0/B:col10/1731690012356/Put/seqid=0 2024-11-15T17:00:12,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742300_1476 (size=12301) 2024-11-15T17:00:12,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:12,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:12,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:12,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:00:13,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690073056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690073065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690073066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690073066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:13,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:13,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/8a7cbe32050e487ebb9f32431f4b291d 2024-11-15T17:00:13,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/d39c9d9375e24fdf9b5436f2e4384163 is 50, key is test_row_0/C:col10/1731690012356/Put/seqid=0 2024-11-15T17:00:13,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742301_1477 (size=12301) 2024-11-15T17:00:13,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:00:13,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:13,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:13,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:13,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690073575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690073575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690073586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690073586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:13,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/d39c9d9375e24fdf9b5436f2e4384163 2024-11-15T17:00:13,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/f692589e09ed467b9e60b16018b5bb79 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/f692589e09ed467b9e60b16018b5bb79 2024-11-15T17:00:13,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/f692589e09ed467b9e60b16018b5bb79, entries=200, sequenceid=276, filesize=14.4 K 2024-11-15T17:00:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/8a7cbe32050e487ebb9f32431f4b291d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/8a7cbe32050e487ebb9f32431f4b291d 2024-11-15T17:00:13,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/8a7cbe32050e487ebb9f32431f4b291d, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T17:00:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/d39c9d9375e24fdf9b5436f2e4384163 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d39c9d9375e24fdf9b5436f2e4384163 2024-11-15T17:00:13,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d39c9d9375e24fdf9b5436f2e4384163, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T17:00:13,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5304b51df6e8f78431676b542bf430ab in 1246ms, sequenceid=276, compaction requested=true 2024-11-15T17:00:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:13,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:13,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:13,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:13,611 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:13,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:13,611 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:13,611 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39723 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:13,611 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:13,611 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:13,612 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:13,612 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,612 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,612 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/9dbe5c231be643fcbe4dbb4e7cdceddc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/4a1ec31657d94d44ba520e41a893c909, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/f692589e09ed467b9e60b16018b5bb79] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=38.8 K 2024-11-15T17:00:13,612 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/90576d8c1b5245218c015f61b20d347f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c58f646218914cbd8772c6471c6252f3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/8a7cbe32050e487ebb9f32431f4b291d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.4 K 2024-11-15T17:00:13,612 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 90576d8c1b5245218c015f61b20d347f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731690008971 2024-11-15T17:00:13,612 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dbe5c231be643fcbe4dbb4e7cdceddc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731690008971 2024-11-15T17:00:13,612 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a1ec31657d94d44ba520e41a893c909, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731690010158 2024-11-15T17:00:13,612 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting c58f646218914cbd8772c6471c6252f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731690010158 2024-11-15T17:00:13,613 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting f692589e09ed467b9e60b16018b5bb79, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731690012356 2024-11-15T17:00:13,613 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7cbe32050e487ebb9f32431f4b291d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731690012356 2024-11-15T17:00:13,620 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:13,620 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:13,620 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/603b407901d141698c0c8d0085c42fc6 is 50, key is test_row_0/B:col10/1731690012356/Put/seqid=0 2024-11-15T17:00:13,620 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/43b7bd74f9524fc293dad4eb3d0b240a is 50, key is test_row_0/A:col10/1731690012356/Put/seqid=0 2024-11-15T17:00:13,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742302_1478 (size=12983) 2024-11-15T17:00:13,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742303_1479 (size=12983) 2024-11-15T17:00:13,628 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/43b7bd74f9524fc293dad4eb3d0b240a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/43b7bd74f9524fc293dad4eb3d0b240a 2024-11-15T17:00:13,632 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into 43b7bd74f9524fc293dad4eb3d0b240a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:13,632 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:13,632 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=13, startTime=1731690013610; duration=0sec 2024-11-15T17:00:13,632 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:13,632 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:13,632 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:13,633 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:13,633 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:13,633 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,634 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/b7c3acedd1aa478993fa621250138579, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/afa78d78d4d14ff79d97836f32031fe5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d39c9d9375e24fdf9b5436f2e4384163] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.4 K 2024-11-15T17:00:13,634 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c3acedd1aa478993fa621250138579, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731690008971 2024-11-15T17:00:13,634 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting afa78d78d4d14ff79d97836f32031fe5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731690010158 2024-11-15T17:00:13,634 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting d39c9d9375e24fdf9b5436f2e4384163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731690012356 2024-11-15T17:00:13,639 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:13,639 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/c864f16116c64ad586155d4e7be9e8e9 is 50, key is test_row_0/C:col10/1731690012356/Put/seqid=0 2024-11-15T17:00:13,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742304_1480 (size=12983) 2024-11-15T17:00:13,647 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/c864f16116c64ad586155d4e7be9e8e9 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/c864f16116c64ad586155d4e7be9e8e9 2024-11-15T17:00:13,651 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into c864f16116c64ad586155d4e7be9e8e9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:13,651 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:13,651 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=13, startTime=1731690013611; duration=0sec 2024-11-15T17:00:13,651 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:13,651 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:13,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T17:00:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:13,696 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:00:13,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/7cb5afa1bc4f4a51984b48c1402ee3f7 is 50, key is test_row_0/A:col10/1731690012412/Put/seqid=0 2024-11-15T17:00:13,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742305_1481 (size=12301) 2024-11-15T17:00:14,028 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/603b407901d141698c0c8d0085c42fc6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/603b407901d141698c0c8d0085c42fc6 2024-11-15T17:00:14,032 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into 603b407901d141698c0c8d0085c42fc6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:14,032 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:14,032 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=13, startTime=1731690013610; duration=0sec 2024-11-15T17:00:14,032 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:14,033 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:14,105 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/7cb5afa1bc4f4a51984b48c1402ee3f7 2024-11-15T17:00:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/fd3e9f845171490ca87bd36c2001edeb is 50, key is test_row_0/B:col10/1731690012412/Put/seqid=0 2024-11-15T17:00:14,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742306_1482 (size=12301) 2024-11-15T17:00:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:00:14,525 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/fd3e9f845171490ca87bd36c2001edeb 2024-11-15T17:00:14,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/e592d084ff2c4ad98cfac454cbfd8e85 is 50, key is test_row_0/C:col10/1731690012412/Put/seqid=0 2024-11-15T17:00:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742307_1483 (size=12301) 2024-11-15T17:00:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:14,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:14,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690074605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690074610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690074611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690074612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690074715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690074726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690074726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690074726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690074925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,938 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/e592d084ff2c4ad98cfac454cbfd8e85 2024-11-15T17:00:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690074936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/7cb5afa1bc4f4a51984b48c1402ee3f7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/7cb5afa1bc4f4a51984b48c1402ee3f7 2024-11-15T17:00:14,950 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/7cb5afa1bc4f4a51984b48c1402ee3f7, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T17:00:14,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/fd3e9f845171490ca87bd36c2001edeb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/fd3e9f845171490ca87bd36c2001edeb 2024-11-15T17:00:14,954 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/fd3e9f845171490ca87bd36c2001edeb, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T17:00:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/e592d084ff2c4ad98cfac454cbfd8e85 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/e592d084ff2c4ad98cfac454cbfd8e85 2024-11-15T17:00:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690074948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690074948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:14,958 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/e592d084ff2c4ad98cfac454cbfd8e85, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T17:00:14,959 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5304b51df6e8f78431676b542bf430ab in 1263ms, sequenceid=303, compaction requested=false 2024-11-15T17:00:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-15T17:00:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-15T17:00:14,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T17:00:14,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-11-15T17:00:14,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.6470 sec 2024-11-15T17:00:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:15,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:15,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:00:15,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/8d272d52720a40bfbc54d85ad07bcb52 is 50, key is test_row_0/A:col10/1731690015239/Put/seqid=0 2024-11-15T17:00:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742308_1484 (size=14741) 2024-11-15T17:00:15,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690075292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690075300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690075301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690075302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690075406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690075406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690075426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690075426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690075616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690075616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690075645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690075645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/8d272d52720a40bfbc54d85ad07bcb52 2024-11-15T17:00:15,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b71808aff29245acad98de9711d1cd64 is 50, key is test_row_0/B:col10/1731690015239/Put/seqid=0 2024-11-15T17:00:15,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742309_1485 (size=12301) 2024-11-15T17:00:15,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690075935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690075935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690075956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690075956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:16,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b71808aff29245acad98de9711d1cd64 2024-11-15T17:00:16,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/645c5f5048154fd9858faba459e89674 is 50, key is test_row_0/C:col10/1731690015239/Put/seqid=0 2024-11-15T17:00:16,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742310_1486 (size=12301) 2024-11-15T17:00:16,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690076445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:16,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:16,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690076455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:16,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690076466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:16,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690076466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:16,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/645c5f5048154fd9858faba459e89674 2024-11-15T17:00:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/8d272d52720a40bfbc54d85ad07bcb52 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8d272d52720a40bfbc54d85ad07bcb52 2024-11-15T17:00:16,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8d272d52720a40bfbc54d85ad07bcb52, entries=200, sequenceid=316, filesize=14.4 K 2024-11-15T17:00:16,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b71808aff29245acad98de9711d1cd64 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b71808aff29245acad98de9711d1cd64 2024-11-15T17:00:16,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b71808aff29245acad98de9711d1cd64, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T17:00:16,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/645c5f5048154fd9858faba459e89674 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/645c5f5048154fd9858faba459e89674 2024-11-15T17:00:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:00:16,485 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:16,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:16,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/645c5f5048154fd9858faba459e89674, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T17:00:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-15T17:00:16,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5304b51df6e8f78431676b542bf430ab in 1246ms, sequenceid=316, compaction requested=true 2024-11-15T17:00:16,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:16,487 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:16,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:16,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:16,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:16,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:16,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:16,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:16,488 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T17:00:16,489 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:16,490 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:16,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:16,492 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:16,492 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:16,492 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:16,492 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:16,492 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:16,492 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:16,492 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/603b407901d141698c0c8d0085c42fc6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/fd3e9f845171490ca87bd36c2001edeb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b71808aff29245acad98de9711d1cd64] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.7 K 2024-11-15T17:00:16,492 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/43b7bd74f9524fc293dad4eb3d0b240a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/7cb5afa1bc4f4a51984b48c1402ee3f7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8d272d52720a40bfbc54d85ad07bcb52] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=39.1 K 2024-11-15T17:00:16,493 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b7bd74f9524fc293dad4eb3d0b240a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731690012356 2024-11-15T17:00:16,493 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cb5afa1bc4f4a51984b48c1402ee3f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731690012407 2024-11-15T17:00:16,493 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d272d52720a40bfbc54d85ad07bcb52, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731690014604 2024-11-15T17:00:16,497 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 603b407901d141698c0c8d0085c42fc6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731690012356 2024-11-15T17:00:16,497 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3e9f845171490ca87bd36c2001edeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731690012407 2024-11-15T17:00:16,497 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting b71808aff29245acad98de9711d1cd64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731690014604 2024-11-15T17:00:16,502 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:16,503 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/2bdb37b2510e494d91afc2dc05e67f83 is 50, key is test_row_0/A:col10/1731690015239/Put/seqid=0 2024-11-15T17:00:16,504 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:16,505 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/36cb123eeb164ec8b8009ee69f5563e2 is 50, key is test_row_0/B:col10/1731690015239/Put/seqid=0 2024-11-15T17:00:16,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742311_1487 (size=13085) 2024-11-15T17:00:16,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742312_1488 (size=13085) 2024-11-15T17:00:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T17:00:16,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-15T17:00:16,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:16,643 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:00:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/54c57d2c94444964adc48c9eb12cf2b7 is 50, key is test_row_0/A:col10/1731690015301/Put/seqid=0 2024-11-15T17:00:16,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742313_1489 (size=12301) 2024-11-15T17:00:16,653 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/54c57d2c94444964adc48c9eb12cf2b7 2024-11-15T17:00:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/5ee8b438ea4f4d49b5268d397647317a is 50, key is test_row_0/B:col10/1731690015301/Put/seqid=0 2024-11-15T17:00:16,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742314_1490 (size=12301) 2024-11-15T17:00:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T17:00:16,914 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/2bdb37b2510e494d91afc2dc05e67f83 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/2bdb37b2510e494d91afc2dc05e67f83 2024-11-15T17:00:16,918 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into 2bdb37b2510e494d91afc2dc05e67f83(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:16,918 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:16,918 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=13, startTime=1731690016487; duration=0sec 2024-11-15T17:00:16,918 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:16,918 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:16,918 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:16,919 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:16,919 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:16,919 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:16,919 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/c864f16116c64ad586155d4e7be9e8e9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/e592d084ff2c4ad98cfac454cbfd8e85, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/645c5f5048154fd9858faba459e89674] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.7 K 2024-11-15T17:00:16,919 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c864f16116c64ad586155d4e7be9e8e9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731690012356 2024-11-15T17:00:16,920 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e592d084ff2c4ad98cfac454cbfd8e85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731690012407 2024-11-15T17:00:16,920 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 645c5f5048154fd9858faba459e89674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731690014604 2024-11-15T17:00:16,925 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:16,925 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/f0b70fee108041e4b611cd8a53b26f79 is 50, key is test_row_0/C:col10/1731690015239/Put/seqid=0 2024-11-15T17:00:16,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742315_1491 (size=13085) 2024-11-15T17:00:16,932 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/36cb123eeb164ec8b8009ee69f5563e2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/36cb123eeb164ec8b8009ee69f5563e2 2024-11-15T17:00:16,937 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into 36cb123eeb164ec8b8009ee69f5563e2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:16,937 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:16,937 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=13, startTime=1731690016488; duration=0sec 2024-11-15T17:00:16,937 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:16,937 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:17,070 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/5ee8b438ea4f4d49b5268d397647317a 2024-11-15T17:00:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/62ec372ffd7c4147945c1e7785982f5f is 50, key is test_row_0/C:col10/1731690015301/Put/seqid=0 2024-11-15T17:00:17,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742316_1492 (size=12301) 2024-11-15T17:00:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T17:00:17,334 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/f0b70fee108041e4b611cd8a53b26f79 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f0b70fee108041e4b611cd8a53b26f79 2024-11-15T17:00:17,338 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into f0b70fee108041e4b611cd8a53b26f79(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:17,339 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:17,339 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=13, startTime=1731690016488; duration=0sec 2024-11-15T17:00:17,339 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:17,339 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:17,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:17,482 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/62ec372ffd7c4147945c1e7785982f5f 2024-11-15T17:00:17,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/54c57d2c94444964adc48c9eb12cf2b7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/54c57d2c94444964adc48c9eb12cf2b7 2024-11-15T17:00:17,489 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/54c57d2c94444964adc48c9eb12cf2b7, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T17:00:17,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/5ee8b438ea4f4d49b5268d397647317a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/5ee8b438ea4f4d49b5268d397647317a 2024-11-15T17:00:17,492 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/5ee8b438ea4f4d49b5268d397647317a, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T17:00:17,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1731690077486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/62ec372ffd7c4147945c1e7785982f5f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/62ec372ffd7c4147945c1e7785982f5f 2024-11-15T17:00:17,493 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 5304b51df6e8f78431676b542bf430ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690077487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690077491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,497 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/62ec372ffd7c4147945c1e7785982f5f, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T17:00:17,498 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5304b51df6e8f78431676b542bf430ab in 855ms, sequenceid=340, compaction requested=false 2024-11-15T17:00:17,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:17,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:17,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-15T17:00:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-15T17:00:17,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-15T17:00:17,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-15T17:00:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:17,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:17,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:00:17,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.0150 sec 2024-11-15T17:00:17,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/ffacf946ac8a474bbcb636b4af7f99ab is 50, key is test_row_0/A:col10/1731690017480/Put/seqid=0 2024-11-15T17:00:17,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742317_1493 (size=14741) 2024-11-15T17:00:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690077611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690077614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690077615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T17:00:17,626 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:17,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-15T17:00:17,627 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:00:17,628 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:17,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690077725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690077725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:00:17,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690077735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:17,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:17,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:17,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:17,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:17,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:17,905 INFO [Thread-1900 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1900. 2024-11-15T17:00:17,905 DEBUG [Thread-1900 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:17,905 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,905 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:17,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/ffacf946ac8a474bbcb636b4af7f99ab 2024-11-15T17:00:17,914 INFO [Thread-1902 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1902. 2024-11-15T17:00:17,914 DEBUG [Thread-1902 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:17,914 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,914 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:17,915 INFO [Thread-1896 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1896. 2024-11-15T17:00:17,916 DEBUG [Thread-1896 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:17,916 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,916 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:17,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/a337b95739204ae4baa6e760c8d1da1b is 50, key is test_row_0/B:col10/1731690017480/Put/seqid=0 2024-11-15T17:00:17,917 INFO [Thread-1894 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1894. 2024-11-15T17:00:17,917 DEBUG [Thread-1894 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:17,917 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,917 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:17,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742318_1494 (size=12301) 2024-11-15T17:00:17,922 INFO [Thread-1898 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1898. 2024-11-15T17:00:17,922 DEBUG [Thread-1898 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:17,922 DEBUG [Thread-1898 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,922 DEBUG [Thread-1898 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:17,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:17,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:17,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:17,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690077935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:00:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690077945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:17,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690077955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:18,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41262 deadline: 1731690078245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:00:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41296 deadline: 1731690078256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41232 deadline: 1731690078266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:18,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/a337b95739204ae4baa6e760c8d1da1b 2024-11-15T17:00:18,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/23ae64341d0542839299108ec50f906c is 50, key is test_row_0/C:col10/1731690017480/Put/seqid=0 2024-11-15T17:00:18,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742319_1495 (size=12301) 2024-11-15T17:00:18,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:18,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:18,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41290 deadline: 1731690078605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:18,606 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 5304b51df6e8f78431676b542bf430ab of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18310 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 , the old value is region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:00:18,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/23ae64341d0542839299108ec50f906c 2024-11-15T17:00:18,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/ffacf946ac8a474bbcb636b4af7f99ab as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/ffacf946ac8a474bbcb636b4af7f99ab 2024-11-15T17:00:18,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/ffacf946ac8a474bbcb636b4af7f99ab, entries=200, sequenceid=356, filesize=14.4 K 2024-11-15T17:00:18,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/a337b95739204ae4baa6e760c8d1da1b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/a337b95739204ae4baa6e760c8d1da1b 2024-11-15T17:00:18,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/a337b95739204ae4baa6e760c8d1da1b, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T17:00:18,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/23ae64341d0542839299108ec50f906c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/23ae64341d0542839299108ec50f906c 2024-11-15T17:00:18,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/23ae64341d0542839299108ec50f906c, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T17:00:18,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5304b51df6e8f78431676b542bf430ab in 1240ms, sequenceid=356, compaction requested=true 2024-11-15T17:00:18,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:18,742 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5304b51df6e8f78431676b542bf430ab:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:18,742 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:18,742 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:18,742 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:18,742 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/A is initiating minor compaction (all files) 2024-11-15T17:00:18,742 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/B is initiating minor compaction (all files) 2024-11-15T17:00:18,742 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/A in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,742 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/B in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,742 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/2bdb37b2510e494d91afc2dc05e67f83, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/54c57d2c94444964adc48c9eb12cf2b7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/ffacf946ac8a474bbcb636b4af7f99ab] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=39.2 K 2024-11-15T17:00:18,743 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/36cb123eeb164ec8b8009ee69f5563e2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/5ee8b438ea4f4d49b5268d397647317a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/a337b95739204ae4baa6e760c8d1da1b] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.8 K 2024-11-15T17:00:18,743 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bdb37b2510e494d91afc2dc05e67f83, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731690014604 2024-11-15T17:00:18,743 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 36cb123eeb164ec8b8009ee69f5563e2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731690014604 2024-11-15T17:00:18,743 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c57d2c94444964adc48c9eb12cf2b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731690015290 2024-11-15T17:00:18,743 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee8b438ea4f4d49b5268d397647317a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731690015290 2024-11-15T17:00:18,743 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffacf946ac8a474bbcb636b4af7f99ab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731690017480 2024-11-15T17:00:18,743 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a337b95739204ae4baa6e760c8d1da1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731690017480 2024-11-15T17:00:18,749 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:18,749 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:18,749 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b1503004e91545ec8fdc59d2f3a1892e is 50, key is test_row_0/B:col10/1731690017480/Put/seqid=0 2024-11-15T17:00:18,749 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/5b03d014b50f406bbbcec7dff155032e is 50, key is test_row_0/A:col10/1731690017480/Put/seqid=0 2024-11-15T17:00:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742320_1496 (size=13187) 2024-11-15T17:00:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742321_1497 (size=13187) 2024-11-15T17:00:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:18,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:18,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:00:18,757 INFO [Thread-1885 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1885. 2024-11-15T17:00:18,757 DEBUG [Thread-1885 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:18,757 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:18,757 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:18,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:18,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c4366dc902014dd1964ae30bd0eb2025 is 50, key is test_row_0/A:col10/1731690017614/Put/seqid=0 2024-11-15T17:00:18,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742322_1498 (size=12301) 2024-11-15T17:00:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:00:18,767 INFO [Thread-1889 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1889. 2024-11-15T17:00:18,767 DEBUG [Thread-1889 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:18,767 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:18,767 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:18,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:18,777 INFO [Thread-1887 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1887. 2024-11-15T17:00:18,777 DEBUG [Thread-1887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:18,777 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:18,777 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:18,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:18,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:18,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:18,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:18,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:19,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:19,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,157 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/5b03d014b50f406bbbcec7dff155032e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/5b03d014b50f406bbbcec7dff155032e 2024-11-15T17:00:19,157 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/b1503004e91545ec8fdc59d2f3a1892e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b1503004e91545ec8fdc59d2f3a1892e 2024-11-15T17:00:19,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c4366dc902014dd1964ae30bd0eb2025 2024-11-15T17:00:19,165 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/A of 5304b51df6e8f78431676b542bf430ab into 5b03d014b50f406bbbcec7dff155032e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:19,165 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/B of 5304b51df6e8f78431676b542bf430ab into b1503004e91545ec8fdc59d2f3a1892e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:19,166 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/A, priority=13, startTime=1731690018742; duration=0sec 2024-11-15T17:00:19,166 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/B, priority=13, startTime=1731690018742; duration=0sec 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:B 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:A 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:19,166 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 5304b51df6e8f78431676b542bf430ab/C is initiating minor compaction (all files) 2024-11-15T17:00:19,167 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5304b51df6e8f78431676b542bf430ab/C in TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,167 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f0b70fee108041e4b611cd8a53b26f79, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/62ec372ffd7c4147945c1e7785982f5f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/23ae64341d0542839299108ec50f906c] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp, totalSize=36.8 K 2024-11-15T17:00:19,167 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f0b70fee108041e4b611cd8a53b26f79, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731690014604 2024-11-15T17:00:19,167 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ec372ffd7c4147945c1e7785982f5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731690015290 2024-11-15T17:00:19,167 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ae64341d0542839299108ec50f906c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731690017480 2024-11-15T17:00:19,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/af8a04983a8a4a179e52a6063d071ec6 is 50, key is test_row_0/B:col10/1731690017614/Put/seqid=0 2024-11-15T17:00:19,173 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5304b51df6e8f78431676b542bf430ab#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:19,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742323_1499 (size=12301) 2024-11-15T17:00:19,174 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/4d26d92927f24ba78e99a1ad0721df87 is 50, key is test_row_0/C:col10/1731690017480/Put/seqid=0 2024-11-15T17:00:19,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742324_1500 (size=13187) 2024-11-15T17:00:19,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,517 INFO [Thread-1891 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1891. 2024-11-15T17:00:19,517 DEBUG [Thread-1891 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:19,517 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:19,517 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:19,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:19,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/af8a04983a8a4a179e52a6063d071ec6 2024-11-15T17:00:19,580 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/4d26d92927f24ba78e99a1ad0721df87 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/4d26d92927f24ba78e99a1ad0721df87 2024-11-15T17:00:19,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/437682289d094fd3b23af0116070745e is 50, key is test_row_0/C:col10/1731690017614/Put/seqid=0 2024-11-15T17:00:19,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742325_1501 (size=12301) 2024-11-15T17:00:19,583 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5304b51df6e8f78431676b542bf430ab/C of 5304b51df6e8f78431676b542bf430ab into 4d26d92927f24ba78e99a1ad0721df87(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:19,583 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:19,583 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., storeName=5304b51df6e8f78431676b542bf430ab/C, priority=13, startTime=1731690018742; duration=0sec 2024-11-15T17:00:19,583 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:19,583 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5304b51df6e8f78431676b542bf430ab:C 2024-11-15T17:00:19,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:19,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:19,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:00:19,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. as already flushing 2024-11-15T17:00:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:19,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:19,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/437682289d094fd3b23af0116070745e 2024-11-15T17:00:19,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/c4366dc902014dd1964ae30bd0eb2025 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c4366dc902014dd1964ae30bd0eb2025 2024-11-15T17:00:19,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c4366dc902014dd1964ae30bd0eb2025, entries=150, sequenceid=380, filesize=12.0 K 2024-11-15T17:00:19,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/af8a04983a8a4a179e52a6063d071ec6 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/af8a04983a8a4a179e52a6063d071ec6 2024-11-15T17:00:19,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/af8a04983a8a4a179e52a6063d071ec6, entries=150, sequenceid=380, filesize=12.0 K 2024-11-15T17:00:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/437682289d094fd3b23af0116070745e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/437682289d094fd3b23af0116070745e 2024-11-15T17:00:19,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/437682289d094fd3b23af0116070745e, entries=150, sequenceid=380, filesize=12.0 K 2024-11-15T17:00:19,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=20.13 KB/20610 for 5304b51df6e8f78431676b542bf430ab in 1238ms, sequenceid=380, compaction requested=false 2024-11-15T17:00:19,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:20,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-15T17:00:20,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:20,072 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-15T17:00:20,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e794b18a26b44240a69fb05283317690 is 50, key is test_row_0/A:col10/1731690018766/Put/seqid=0 2024-11-15T17:00:20,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742326_1502 (size=12301) 2024-11-15T17:00:20,480 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e794b18a26b44240a69fb05283317690 2024-11-15T17:00:20,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/ff91c2a5c4ab48c592e6664434fb7ab5 is 50, key is test_row_0/B:col10/1731690018766/Put/seqid=0 2024-11-15T17:00:20,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742327_1503 (size=12301) 2024-11-15T17:00:20,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:00:20,890 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/ff91c2a5c4ab48c592e6664434fb7ab5 2024-11-15T17:00:20,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/7b72fb1f56fa44febe15fdf7f2d9c782 is 50, key is test_row_0/C:col10/1731690018766/Put/seqid=0 2024-11-15T17:00:20,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742328_1504 (size=12301) 2024-11-15T17:00:21,299 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/7b72fb1f56fa44febe15fdf7f2d9c782 2024-11-15T17:00:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/e794b18a26b44240a69fb05283317690 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e794b18a26b44240a69fb05283317690 2024-11-15T17:00:21,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e794b18a26b44240a69fb05283317690, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T17:00:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/ff91c2a5c4ab48c592e6664434fb7ab5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/ff91c2a5c4ab48c592e6664434fb7ab5 2024-11-15T17:00:21,308 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/ff91c2a5c4ab48c592e6664434fb7ab5, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T17:00:21,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/7b72fb1f56fa44febe15fdf7f2d9c782 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/7b72fb1f56fa44febe15fdf7f2d9c782 2024-11-15T17:00:21,311 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/7b72fb1f56fa44febe15fdf7f2d9c782, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T17:00:21,311 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 5304b51df6e8f78431676b542bf430ab in 1239ms, sequenceid=389, compaction requested=true 2024-11-15T17:00:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 5304b51df6e8f78431676b542bf430ab: 2024-11-15T17:00:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-15T17:00:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-15T17:00:21,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-15T17:00:21,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6850 sec 2024-11-15T17:00:21,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 3.6890 sec 2024-11-15T17:00:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:00:21,796 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:28,617 INFO [Thread-1883 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1883. 2024-11-15T17:00:28,617 DEBUG [Thread-1883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:28,617 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:28,617 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:28,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1469 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4404 rows 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1461 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4383 rows 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1470 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4410 rows 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1462 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4386 rows 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1471 2024-11-15T17:00:28,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4413 rows 2024-11-15T17:00:28,618 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:00:28,618 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:00:28,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:28,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:28,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:28,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:00:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-15T17:00:28,621 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731690028621"}]},"ts":"1731690028621"} 2024-11-15T17:00:28,622 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:00:28,623 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:00:28,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:00:28,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, UNASSIGN}] 2024-11-15T17:00:28,625 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, UNASSIGN 2024-11-15T17:00:28,625 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5304b51df6e8f78431676b542bf430ab, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:28,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, UNASSIGN because future has completed 2024-11-15T17:00:28,627 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:00:28,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T17:00:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-15T17:00:28,779 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:28,779 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:00:28,779 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 5304b51df6e8f78431676b542bf430ab, disabling compactions & flushes 2024-11-15T17:00:28,779 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:28,779 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:28,779 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. after waiting 0 ms 2024-11-15T17:00:28,779 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:28,779 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing 5304b51df6e8f78431676b542bf430ab 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-15T17:00:28,783 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/8c7df48967fd4a0a86a5363a9de19988 is 50, key is test_row_1/A:col10/1731690028616/Put/seqid=0 2024-11-15T17:00:28,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742329_1505 (size=7415) 2024-11-15T17:00:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-15T17:00:29,186 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/8c7df48967fd4a0a86a5363a9de19988 2024-11-15T17:00:29,192 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/470e2dd37d0649b5b3cdc82bff8a5a9f is 50, key is test_row_1/B:col10/1731690028616/Put/seqid=0 2024-11-15T17:00:29,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742330_1506 (size=7415) 2024-11-15T17:00:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-15T17:00:29,596 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/470e2dd37d0649b5b3cdc82bff8a5a9f 2024-11-15T17:00:29,601 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/90c10c9a034c425d822697c421beb0c9 is 50, key is test_row_1/C:col10/1731690028616/Put/seqid=0 2024-11-15T17:00:29,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742331_1507 (size=7415) 2024-11-15T17:00:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-15T17:00:30,005 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/90c10c9a034c425d822697c421beb0c9 2024-11-15T17:00:30,008 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/A/8c7df48967fd4a0a86a5363a9de19988 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8c7df48967fd4a0a86a5363a9de19988 2024-11-15T17:00:30,011 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8c7df48967fd4a0a86a5363a9de19988, entries=50, sequenceid=393, filesize=7.2 K 2024-11-15T17:00:30,011 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/B/470e2dd37d0649b5b3cdc82bff8a5a9f as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/470e2dd37d0649b5b3cdc82bff8a5a9f 2024-11-15T17:00:30,014 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/470e2dd37d0649b5b3cdc82bff8a5a9f, entries=50, sequenceid=393, filesize=7.2 K 2024-11-15T17:00:30,014 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/.tmp/C/90c10c9a034c425d822697c421beb0c9 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/90c10c9a034c425d822697c421beb0c9 2024-11-15T17:00:30,017 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/90c10c9a034c425d822697c421beb0c9, entries=50, sequenceid=393, filesize=7.2 K 2024-11-15T17:00:30,017 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5304b51df6e8f78431676b542bf430ab in 1238ms, sequenceid=393, compaction requested=true 2024-11-15T17:00:30,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c038b334e2574753b329d8884d71e9cf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e83d60cf77a546dc9f764e9ab207b1ed, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c671c2b8c32b4d8282f1fd4bd5097466, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/50d1188714f34ffba96d8b138ef52044, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/25a5447462e648bdbae2e532ffa1c272, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/72893ee5aa0b4aa987c38f5014045daf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/fb9ab7a5451648ddb9369d658f854f7c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a8b0722758484a889ccd799945572fbc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/45873a20e0bb435db023a1eb221db9e7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/90f652ec78cc4d66b2f7a5cf109e3a3d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e4aa699cf7694c55bd84bf359fa4de41, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/d98eb169154a4ff4abefaf605a962aaf, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e17d08a607b8495f98859faff2a90e53, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c5c692de3c3248a6a1cdd1f973a5efa0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a33852f195524dfc83465384b1205c52, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/16125b9ec7fd4f5caf4ebac6044b1972, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e5f5b2bfc7f6409f9abbee2887aec199, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/9dbe5c231be643fcbe4dbb4e7cdceddc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/4a1ec31657d94d44ba520e41a893c909, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/f692589e09ed467b9e60b16018b5bb79, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/43b7bd74f9524fc293dad4eb3d0b240a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/7cb5afa1bc4f4a51984b48c1402ee3f7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8d272d52720a40bfbc54d85ad07bcb52, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/2bdb37b2510e494d91afc2dc05e67f83, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/54c57d2c94444964adc48c9eb12cf2b7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/ffacf946ac8a474bbcb636b4af7f99ab] to archive 2024-11-15T17:00:30,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:00:30,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c038b334e2574753b329d8884d71e9cf to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c038b334e2574753b329d8884d71e9cf 2024-11-15T17:00:30,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e83d60cf77a546dc9f764e9ab207b1ed to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e83d60cf77a546dc9f764e9ab207b1ed 2024-11-15T17:00:30,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c671c2b8c32b4d8282f1fd4bd5097466 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c671c2b8c32b4d8282f1fd4bd5097466 2024-11-15T17:00:30,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/50d1188714f34ffba96d8b138ef52044 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/50d1188714f34ffba96d8b138ef52044 2024-11-15T17:00:30,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/25a5447462e648bdbae2e532ffa1c272 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/25a5447462e648bdbae2e532ffa1c272 2024-11-15T17:00:30,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/72893ee5aa0b4aa987c38f5014045daf to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/72893ee5aa0b4aa987c38f5014045daf 2024-11-15T17:00:30,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/fb9ab7a5451648ddb9369d658f854f7c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/fb9ab7a5451648ddb9369d658f854f7c 2024-11-15T17:00:30,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a8b0722758484a889ccd799945572fbc to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a8b0722758484a889ccd799945572fbc 2024-11-15T17:00:30,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/45873a20e0bb435db023a1eb221db9e7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/45873a20e0bb435db023a1eb221db9e7 2024-11-15T17:00:30,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/90f652ec78cc4d66b2f7a5cf109e3a3d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/90f652ec78cc4d66b2f7a5cf109e3a3d 2024-11-15T17:00:30,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e4aa699cf7694c55bd84bf359fa4de41 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e4aa699cf7694c55bd84bf359fa4de41 2024-11-15T17:00:30,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/d98eb169154a4ff4abefaf605a962aaf to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/d98eb169154a4ff4abefaf605a962aaf 2024-11-15T17:00:30,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e17d08a607b8495f98859faff2a90e53 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e17d08a607b8495f98859faff2a90e53 2024-11-15T17:00:30,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c5c692de3c3248a6a1cdd1f973a5efa0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c5c692de3c3248a6a1cdd1f973a5efa0 2024-11-15T17:00:30,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a33852f195524dfc83465384b1205c52 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/a33852f195524dfc83465384b1205c52 2024-11-15T17:00:30,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/16125b9ec7fd4f5caf4ebac6044b1972 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/16125b9ec7fd4f5caf4ebac6044b1972 2024-11-15T17:00:30,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e5f5b2bfc7f6409f9abbee2887aec199 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e5f5b2bfc7f6409f9abbee2887aec199 2024-11-15T17:00:30,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/9dbe5c231be643fcbe4dbb4e7cdceddc to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/9dbe5c231be643fcbe4dbb4e7cdceddc 2024-11-15T17:00:30,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/4a1ec31657d94d44ba520e41a893c909 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/4a1ec31657d94d44ba520e41a893c909 2024-11-15T17:00:30,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/f692589e09ed467b9e60b16018b5bb79 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/f692589e09ed467b9e60b16018b5bb79 2024-11-15T17:00:30,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/43b7bd74f9524fc293dad4eb3d0b240a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/43b7bd74f9524fc293dad4eb3d0b240a 2024-11-15T17:00:30,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/7cb5afa1bc4f4a51984b48c1402ee3f7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/7cb5afa1bc4f4a51984b48c1402ee3f7 2024-11-15T17:00:30,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8d272d52720a40bfbc54d85ad07bcb52 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8d272d52720a40bfbc54d85ad07bcb52 2024-11-15T17:00:30,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/2bdb37b2510e494d91afc2dc05e67f83 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/2bdb37b2510e494d91afc2dc05e67f83 2024-11-15T17:00:30,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/54c57d2c94444964adc48c9eb12cf2b7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/54c57d2c94444964adc48c9eb12cf2b7 2024-11-15T17:00:30,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/ffacf946ac8a474bbcb636b4af7f99ab to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/ffacf946ac8a474bbcb636b4af7f99ab 2024-11-15T17:00:30,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/f96606ae9a5f4b24a5d666ef1a070bf2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/edd5d322cc3049338ff6812941c53b6f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/e78a7858ad6e4caea63f95fa38755622, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/998c8bbaf8474ffe9f10cb04cb88cfcb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/6c77bc3eef4944099089b74205ede63f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/0af8e04c29e54b3780c7cf6370dccc84, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/cc9b6cc7d49c499d9e4c760ed1fc20a0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/23408a7b737445b18156592f76e1ce49, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/7f169809a95f439099a4b579f90206e3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/d731731b95d14bc485f927202f072a69, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/42d66ce8f98e4a13bacfdc20205b6877, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b2519d891bd64948b97bdf6c8d8109f4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/10cb9cd795284838b3ab466b6ebbdf4c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/1aedd2fdbed14e2db3fc2e154a2fec25, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/218adbb2b1664ab981c09fff04fee175, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c53f98cfccc44bd585ed386f2d296847, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/90576d8c1b5245218c015f61b20d347f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/9c9221df69ac4c33a7f82364078649ef, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c58f646218914cbd8772c6471c6252f3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/603b407901d141698c0c8d0085c42fc6, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/8a7cbe32050e487ebb9f32431f4b291d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/fd3e9f845171490ca87bd36c2001edeb, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/36cb123eeb164ec8b8009ee69f5563e2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b71808aff29245acad98de9711d1cd64, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/5ee8b438ea4f4d49b5268d397647317a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/a337b95739204ae4baa6e760c8d1da1b] to archive 2024-11-15T17:00:30,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:00:30,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/f96606ae9a5f4b24a5d666ef1a070bf2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/f96606ae9a5f4b24a5d666ef1a070bf2 2024-11-15T17:00:30,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/edd5d322cc3049338ff6812941c53b6f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/edd5d322cc3049338ff6812941c53b6f 2024-11-15T17:00:30,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/e78a7858ad6e4caea63f95fa38755622 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/e78a7858ad6e4caea63f95fa38755622 2024-11-15T17:00:30,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/998c8bbaf8474ffe9f10cb04cb88cfcb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/998c8bbaf8474ffe9f10cb04cb88cfcb 2024-11-15T17:00:30,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/6c77bc3eef4944099089b74205ede63f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/6c77bc3eef4944099089b74205ede63f 2024-11-15T17:00:30,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/0af8e04c29e54b3780c7cf6370dccc84 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/0af8e04c29e54b3780c7cf6370dccc84 2024-11-15T17:00:30,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/cc9b6cc7d49c499d9e4c760ed1fc20a0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/cc9b6cc7d49c499d9e4c760ed1fc20a0 2024-11-15T17:00:30,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/23408a7b737445b18156592f76e1ce49 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/23408a7b737445b18156592f76e1ce49 2024-11-15T17:00:30,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/7f169809a95f439099a4b579f90206e3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/7f169809a95f439099a4b579f90206e3 2024-11-15T17:00:30,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/d731731b95d14bc485f927202f072a69 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/d731731b95d14bc485f927202f072a69 2024-11-15T17:00:30,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/42d66ce8f98e4a13bacfdc20205b6877 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/42d66ce8f98e4a13bacfdc20205b6877 2024-11-15T17:00:30,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b2519d891bd64948b97bdf6c8d8109f4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b2519d891bd64948b97bdf6c8d8109f4 2024-11-15T17:00:30,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/10cb9cd795284838b3ab466b6ebbdf4c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/10cb9cd795284838b3ab466b6ebbdf4c 2024-11-15T17:00:30,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/1aedd2fdbed14e2db3fc2e154a2fec25 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/1aedd2fdbed14e2db3fc2e154a2fec25 2024-11-15T17:00:30,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/218adbb2b1664ab981c09fff04fee175 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/218adbb2b1664ab981c09fff04fee175 2024-11-15T17:00:30,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c53f98cfccc44bd585ed386f2d296847 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c53f98cfccc44bd585ed386f2d296847 2024-11-15T17:00:30,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/90576d8c1b5245218c015f61b20d347f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/90576d8c1b5245218c015f61b20d347f 2024-11-15T17:00:30,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/9c9221df69ac4c33a7f82364078649ef to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/9c9221df69ac4c33a7f82364078649ef 2024-11-15T17:00:30,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c58f646218914cbd8772c6471c6252f3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/c58f646218914cbd8772c6471c6252f3 2024-11-15T17:00:30,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/603b407901d141698c0c8d0085c42fc6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/603b407901d141698c0c8d0085c42fc6 2024-11-15T17:00:30,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/8a7cbe32050e487ebb9f32431f4b291d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/8a7cbe32050e487ebb9f32431f4b291d 2024-11-15T17:00:30,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/fd3e9f845171490ca87bd36c2001edeb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/fd3e9f845171490ca87bd36c2001edeb 2024-11-15T17:00:30,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/36cb123eeb164ec8b8009ee69f5563e2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/36cb123eeb164ec8b8009ee69f5563e2 2024-11-15T17:00:30,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b71808aff29245acad98de9711d1cd64 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b71808aff29245acad98de9711d1cd64 2024-11-15T17:00:30,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/5ee8b438ea4f4d49b5268d397647317a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/5ee8b438ea4f4d49b5268d397647317a 2024-11-15T17:00:30,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/a337b95739204ae4baa6e760c8d1da1b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/a337b95739204ae4baa6e760c8d1da1b 2024-11-15T17:00:30,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/864b02a6dce04309ba8d84a5c2e78c5d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/83fad30f349644aba855957bc18a6ce7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/33d42da1337a49288adf1d675e62d22f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d9ba4fca91534634b407aa5edc6bef39, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/a4f5949f9fc9428286bfa087352ec2fc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/abf13351a05942b8b7bd1337b938ea60, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/1abedcb6223d4fe7a1f070704bf2ae58, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f60218f34fb84e049c6de5870fb74b81, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3bd89f22362e43a69a8348cc98f71e76, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dbecbb9982ae48e0b6667ef2685b77fa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/5c9715853cee405d8d8744e2127ba6ea, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/586abdf12a26482c94e44f6298dc1b59, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/04154c3f22ed49c9b737cb27c9bec7e8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/ca652bd0b67b49c796b88a87fbd56ca3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dccae92ed671465fb444f058766c4043, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3d1e3d7de00f4b4090267e0dddd92c1a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/b7c3acedd1aa478993fa621250138579, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/12505430417948e898c51540f3ecd4e2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/afa78d78d4d14ff79d97836f32031fe5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/c864f16116c64ad586155d4e7be9e8e9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d39c9d9375e24fdf9b5436f2e4384163, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/e592d084ff2c4ad98cfac454cbfd8e85, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f0b70fee108041e4b611cd8a53b26f79, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/645c5f5048154fd9858faba459e89674, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/62ec372ffd7c4147945c1e7785982f5f, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/23ae64341d0542839299108ec50f906c] to archive 2024-11-15T17:00:30,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:00:30,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/864b02a6dce04309ba8d84a5c2e78c5d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/864b02a6dce04309ba8d84a5c2e78c5d 2024-11-15T17:00:30,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/83fad30f349644aba855957bc18a6ce7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/83fad30f349644aba855957bc18a6ce7 2024-11-15T17:00:30,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/33d42da1337a49288adf1d675e62d22f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/33d42da1337a49288adf1d675e62d22f 2024-11-15T17:00:30,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d9ba4fca91534634b407aa5edc6bef39 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d9ba4fca91534634b407aa5edc6bef39 2024-11-15T17:00:30,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/a4f5949f9fc9428286bfa087352ec2fc to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/a4f5949f9fc9428286bfa087352ec2fc 2024-11-15T17:00:30,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/abf13351a05942b8b7bd1337b938ea60 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/abf13351a05942b8b7bd1337b938ea60 2024-11-15T17:00:30,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/1abedcb6223d4fe7a1f070704bf2ae58 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/1abedcb6223d4fe7a1f070704bf2ae58 2024-11-15T17:00:30,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f60218f34fb84e049c6de5870fb74b81 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f60218f34fb84e049c6de5870fb74b81 2024-11-15T17:00:30,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3bd89f22362e43a69a8348cc98f71e76 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3bd89f22362e43a69a8348cc98f71e76 2024-11-15T17:00:30,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dbecbb9982ae48e0b6667ef2685b77fa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dbecbb9982ae48e0b6667ef2685b77fa 2024-11-15T17:00:30,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/5c9715853cee405d8d8744e2127ba6ea to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/5c9715853cee405d8d8744e2127ba6ea 2024-11-15T17:00:30,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/586abdf12a26482c94e44f6298dc1b59 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/586abdf12a26482c94e44f6298dc1b59 2024-11-15T17:00:30,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/04154c3f22ed49c9b737cb27c9bec7e8 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/04154c3f22ed49c9b737cb27c9bec7e8 2024-11-15T17:00:30,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/ca652bd0b67b49c796b88a87fbd56ca3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/ca652bd0b67b49c796b88a87fbd56ca3 2024-11-15T17:00:30,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dccae92ed671465fb444f058766c4043 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/dccae92ed671465fb444f058766c4043 2024-11-15T17:00:30,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3d1e3d7de00f4b4090267e0dddd92c1a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/3d1e3d7de00f4b4090267e0dddd92c1a 2024-11-15T17:00:30,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/b7c3acedd1aa478993fa621250138579 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/b7c3acedd1aa478993fa621250138579 2024-11-15T17:00:30,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/12505430417948e898c51540f3ecd4e2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/12505430417948e898c51540f3ecd4e2 2024-11-15T17:00:30,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/afa78d78d4d14ff79d97836f32031fe5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/afa78d78d4d14ff79d97836f32031fe5 2024-11-15T17:00:30,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/c864f16116c64ad586155d4e7be9e8e9 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/c864f16116c64ad586155d4e7be9e8e9 2024-11-15T17:00:30,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d39c9d9375e24fdf9b5436f2e4384163 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/d39c9d9375e24fdf9b5436f2e4384163 2024-11-15T17:00:30,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/e592d084ff2c4ad98cfac454cbfd8e85 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/e592d084ff2c4ad98cfac454cbfd8e85 2024-11-15T17:00:30,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f0b70fee108041e4b611cd8a53b26f79 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/f0b70fee108041e4b611cd8a53b26f79 2024-11-15T17:00:30,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/645c5f5048154fd9858faba459e89674 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/645c5f5048154fd9858faba459e89674 2024-11-15T17:00:30,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/62ec372ffd7c4147945c1e7785982f5f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/62ec372ffd7c4147945c1e7785982f5f 2024-11-15T17:00:30,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/23ae64341d0542839299108ec50f906c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/23ae64341d0542839299108ec50f906c 2024-11-15T17:00:30,088 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=1 2024-11-15T17:00:30,088 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. 2024-11-15T17:00:30,088 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 5304b51df6e8f78431676b542bf430ab: Waiting for close lock at 1731690028779Running coprocessor pre-close hooks at 1731690028779Disabling compacts and flushes for region at 1731690028779Disabling writes for close at 1731690028779Obtaining lock to block concurrent updates at 1731690028779Preparing flush snapshotting stores in 5304b51df6e8f78431676b542bf430ab at 1731690028779Finished memstore snapshotting TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731690028780 (+1 ms)Flushing stores of TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab. at 1731690028780Flushing 5304b51df6e8f78431676b542bf430ab/A: creating writer at 1731690028780Flushing 5304b51df6e8f78431676b542bf430ab/A: appending metadata at 1731690028782 (+2 ms)Flushing 5304b51df6e8f78431676b542bf430ab/A: closing flushed file at 1731690028782Flushing 5304b51df6e8f78431676b542bf430ab/B: creating writer at 1731690029190 (+408 ms)Flushing 5304b51df6e8f78431676b542bf430ab/B: appending metadata at 1731690029192 (+2 ms)Flushing 5304b51df6e8f78431676b542bf430ab/B: closing flushed file at 1731690029192Flushing 5304b51df6e8f78431676b542bf430ab/C: creating writer at 1731690029599 (+407 ms)Flushing 5304b51df6e8f78431676b542bf430ab/C: appending metadata at 1731690029601 (+2 ms)Flushing 5304b51df6e8f78431676b542bf430ab/C: closing flushed file at 1731690029601Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f13a7aa: reopening flushed file at 1731690030008 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10045b27: reopening flushed file at 1731690030011 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b0af9d8: reopening flushed file at 1731690030014 (+3 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5304b51df6e8f78431676b542bf430ab in 1238ms, sequenceid=393, compaction requested=true at 1731690030017 (+3 ms)Writing region close event to WAL at 1731690030085 (+68 ms)Running coprocessor post-close hooks at 1731690030088 (+3 ms)Closed at 1731690030088 2024-11-15T17:00:30,090 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:30,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5304b51df6e8f78431676b542bf430ab, regionState=CLOSED 2024-11-15T17:00:30,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T17:00:30,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-15T17:00:30,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 5304b51df6e8f78431676b542bf430ab, server=a9bde5b3a1b5,35119,1731689872090 in 1.4650 sec 2024-11-15T17:00:30,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-15T17:00:30,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5304b51df6e8f78431676b542bf430ab, UNASSIGN in 1.4690 sec 2024-11-15T17:00:30,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-15T17:00:30,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4720 sec 2024-11-15T17:00:30,097 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731690030097"}]},"ts":"1731690030097"} 2024-11-15T17:00:30,098 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:00:30,098 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:00:30,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-15T17:00:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-15T17:00:30,765 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:30,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:00:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:30,768 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-15T17:00:30,768 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:30,771 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:30,772 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/recovered.edits] 2024-11-15T17:00:30,774 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/5b03d014b50f406bbbcec7dff155032e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/5b03d014b50f406bbbcec7dff155032e 2024-11-15T17:00:30,775 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8c7df48967fd4a0a86a5363a9de19988 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/8c7df48967fd4a0a86a5363a9de19988 2024-11-15T17:00:30,776 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c4366dc902014dd1964ae30bd0eb2025 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/c4366dc902014dd1964ae30bd0eb2025 2024-11-15T17:00:30,777 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e794b18a26b44240a69fb05283317690 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/A/e794b18a26b44240a69fb05283317690 2024-11-15T17:00:30,778 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/470e2dd37d0649b5b3cdc82bff8a5a9f to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/470e2dd37d0649b5b3cdc82bff8a5a9f 2024-11-15T17:00:30,779 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/af8a04983a8a4a179e52a6063d071ec6 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/af8a04983a8a4a179e52a6063d071ec6 2024-11-15T17:00:30,780 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b1503004e91545ec8fdc59d2f3a1892e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/b1503004e91545ec8fdc59d2f3a1892e 2024-11-15T17:00:30,781 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/ff91c2a5c4ab48c592e6664434fb7ab5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/B/ff91c2a5c4ab48c592e6664434fb7ab5 2024-11-15T17:00:30,782 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/437682289d094fd3b23af0116070745e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/437682289d094fd3b23af0116070745e 2024-11-15T17:00:30,783 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/4d26d92927f24ba78e99a1ad0721df87 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/4d26d92927f24ba78e99a1ad0721df87 2024-11-15T17:00:30,784 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/7b72fb1f56fa44febe15fdf7f2d9c782 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/7b72fb1f56fa44febe15fdf7f2d9c782 2024-11-15T17:00:30,785 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/90c10c9a034c425d822697c421beb0c9 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/C/90c10c9a034c425d822697c421beb0c9 2024-11-15T17:00:30,787 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/recovered.edits/396.seqid to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab/recovered.edits/396.seqid 2024-11-15T17:00:30,787 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/5304b51df6e8f78431676b542bf430ab 2024-11-15T17:00:30,787 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:00:30,789 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:30,791 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:00:30,793 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:00:30,794 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:30,794 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:00:30,795 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731690030794"}]},"ts":"9223372036854775807"} 2024-11-15T17:00:30,796 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:00:30,796 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5304b51df6e8f78431676b542bf430ab, NAME => 'TestAcidGuarantees,,1731689995708.5304b51df6e8f78431676b542bf430ab.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:00:30,797 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:00:30,797 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731690030797"}]},"ts":"9223372036854775807"} 2024-11-15T17:00:30,798 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:00:30,799 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:30,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-15T17:00:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-15T17:00:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:00:30,877 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:30,889 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 238), OpenFileDescriptor=443 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=252 (was 283), ProcessCount=11 (was 11), AvailableMemoryMB=5214 (was 5286) 2024-11-15T17:00:30,899 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=252, ProcessCount=11, AvailableMemoryMB=5214 2024-11-15T17:00:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:00:30,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:00:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:30,903 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:00:30,903 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:30,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-11-15T17:00:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T17:00:30,904 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:00:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742332_1508 (size=959) 2024-11-15T17:00:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T17:00:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T17:00:31,312 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a 2024-11-15T17:00:31,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742333_1509 (size=53) 2024-11-15T17:00:31,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:00:31,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 696d98f18be792cdc1435b3b0e2fd21e, disabling compactions & flushes 2024-11-15T17:00:31,319 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:31,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:31,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. after waiting 0 ms 2024-11-15T17:00:31,320 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:31,320 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:31,320 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 696d98f18be792cdc1435b3b0e2fd21e: Waiting for close lock at 1731690031319Disabling compacts and flushes for region at 1731690031319Disabling writes for close at 1731690031319Writing region close event to WAL at 1731690031320 (+1 ms)Closed at 1731690031320 2024-11-15T17:00:31,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:00:31,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731690031321"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731690031321"}]},"ts":"1731690031321"} 2024-11-15T17:00:31,323 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:00:31,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:00:31,324 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731690031323"}]},"ts":"1731690031323"} 2024-11-15T17:00:31,325 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T17:00:31,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, ASSIGN}] 2024-11-15T17:00:31,326 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, ASSIGN 2024-11-15T17:00:31,327 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, ASSIGN; state=OFFLINE, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=false 2024-11-15T17:00:31,478 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:31,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, ASSIGN because future has completed 2024-11-15T17:00:31,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T17:00:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T17:00:31,633 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:31,633 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:00:31,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:00:31,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,634 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,635 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,636 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696d98f18be792cdc1435b3b0e2fd21e columnFamilyName A 2024-11-15T17:00:31,636 DEBUG [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:31,637 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(327): Store=696d98f18be792cdc1435b3b0e2fd21e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:00:31,637 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,638 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696d98f18be792cdc1435b3b0e2fd21e columnFamilyName B 2024-11-15T17:00:31,638 DEBUG [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:31,638 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(327): Store=696d98f18be792cdc1435b3b0e2fd21e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:00:31,638 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,639 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696d98f18be792cdc1435b3b0e2fd21e columnFamilyName C 2024-11-15T17:00:31,639 DEBUG [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:31,639 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(327): Store=696d98f18be792cdc1435b3b0e2fd21e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:00:31,639 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,640 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,640 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,641 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,641 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,641 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:00:31,642 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,643 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:00:31,644 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened 696d98f18be792cdc1435b3b0e2fd21e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61769070, jitterRate=-0.07956913113594055}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:00:31,644 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:31,644 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for 696d98f18be792cdc1435b3b0e2fd21e: Running coprocessor pre-open hook at 1731690031634Writing region info on filesystem at 1731690031634Initializing all the Stores at 1731690031634Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731690031635 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731690031635Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731690031635Cleaning up temporary data from old regions at 1731690031641 (+6 ms)Running coprocessor post-open hooks at 1731690031644 (+3 ms)Region opened successfully at 1731690031644 2024-11-15T17:00:31,645 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., pid=152, masterSystemTime=1731690031631 2024-11-15T17:00:31,646 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:31,646 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:31,647 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=OPEN, openSeqNum=2, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:31,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T17:00:31,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-15T17:00:31,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 in 170 msec 2024-11-15T17:00:31,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-15T17:00:31,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, ASSIGN in 325 msec 2024-11-15T17:00:31,652 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:00:31,652 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731690031652"}]},"ts":"1731690031652"} 2024-11-15T17:00:31,653 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T17:00:31,654 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:00:31,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 753 msec 2024-11-15T17:00:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T17:00:32,045 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:32,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,048 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a683b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e1a326b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:32,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:32,052 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:00:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is a9bde5b3a1b5,41785,1731689871395 2024-11-15T17:00:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3145866a 2024-11-15T17:00:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:00:32,054 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:00:32,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:00:32,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:00:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T17:00:32,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742334_1510 (size=995) 2024-11-15T17:00:32,467 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T17:00:32,467 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T17:00:32,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:00:32,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, REOPEN/MOVE}] 2024-11-15T17:00:32,472 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, REOPEN/MOVE 2024-11-15T17:00:32,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:32,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, REOPEN/MOVE because future has completed 2024-11-15T17:00:32,475 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:00:32,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T17:00:32,628 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,628 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:00:32,628 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 696d98f18be792cdc1435b3b0e2fd21e, disabling compactions & flushes 2024-11-15T17:00:32,628 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:32,628 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:32,628 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. after waiting 0 ms 2024-11-15T17:00:32,628 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:32,632 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T17:00:32,632 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:32,632 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 696d98f18be792cdc1435b3b0e2fd21e: Waiting for close lock at 1731690032628Running coprocessor pre-close hooks at 1731690032628Disabling compacts and flushes for region at 1731690032628Disabling writes for close at 1731690032628Writing region close event to WAL at 1731690032628Running coprocessor post-close hooks at 1731690032632 (+4 ms)Closed at 1731690032632 2024-11-15T17:00:32,632 WARN [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: 696d98f18be792cdc1435b3b0e2fd21e to self. 2024-11-15T17:00:32,634 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,634 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=CLOSED 2024-11-15T17:00:32,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T17:00:32,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-15T17:00:32,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 in 161 msec 2024-11-15T17:00:32,639 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, REOPEN/MOVE; state=CLOSED, location=a9bde5b3a1b5,35119,1731689872090; forceNewPlan=false, retain=true 2024-11-15T17:00:32,789 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=OPENING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:32,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, REOPEN/MOVE because future has completed 2024-11-15T17:00:32,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T17:00:32,947 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:32,947 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:00:32,947 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,947 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:00:32,947 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,947 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,949 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,949 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696d98f18be792cdc1435b3b0e2fd21e columnFamilyName A 2024-11-15T17:00:32,951 DEBUG [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:32,951 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(327): Store=696d98f18be792cdc1435b3b0e2fd21e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:00:32,951 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,952 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696d98f18be792cdc1435b3b0e2fd21e columnFamilyName B 2024-11-15T17:00:32,952 DEBUG [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:32,952 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(327): Store=696d98f18be792cdc1435b3b0e2fd21e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:00:32,952 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,953 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 696d98f18be792cdc1435b3b0e2fd21e columnFamilyName C 2024-11-15T17:00:32,953 DEBUG [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:32,953 INFO [StoreOpener-696d98f18be792cdc1435b3b0e2fd21e-1 {}] regionserver.HStore(327): Store=696d98f18be792cdc1435b3b0e2fd21e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:00:32,953 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,954 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,955 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,956 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,956 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,956 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:00:32,958 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,958 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened 696d98f18be792cdc1435b3b0e2fd21e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64839159, jitterRate=-0.033821240067481995}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:00:32,959 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:32,959 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for 696d98f18be792cdc1435b3b0e2fd21e: Running coprocessor pre-open hook at 1731690032947Writing region info on filesystem at 1731690032947Initializing all the Stores at 1731690032948 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731690032948Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731690032948Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731690032948Cleaning up temporary data from old regions at 1731690032956 (+8 ms)Running coprocessor post-open hooks at 1731690032959 (+3 ms)Region opened successfully at 1731690032959 2024-11-15T17:00:32,960 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., pid=157, masterSystemTime=1731690032944 2024-11-15T17:00:32,962 DEBUG [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:32,962 INFO [RS_OPEN_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:32,962 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=OPEN, openSeqNum=5, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:32,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T17:00:32,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-11-15T17:00:32,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 in 173 msec 2024-11-15T17:00:32,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-15T17:00:32,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, REOPEN/MOVE in 495 msec 2024-11-15T17:00:32,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-15T17:00:32,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-15T17:00:32,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-15T17:00:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-15T17:00:32,974 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:32,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc95246, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,977 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405f0a48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,979 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e739c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,981 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169d1a2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f7d8b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,984 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6171be8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,986 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a973c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d43b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,988 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472ce56d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f050946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739501a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375a2168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:32,996 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ab665f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:32,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b35e38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:32,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:32,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:33,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195a318a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:33,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35cfefce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:33,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:33,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538fbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:33,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fcb670b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a9bde5b3a1b5,41785,-1 for getting cluster id 2024-11-15T17:00:33,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:00:33,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f8d877-313a-4ca6-8748-58e5181fa98a' 2024-11-15T17:00:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:00:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f8d877-313a-4ca6-8748-58e5181fa98a" 2024-11-15T17:00:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156411c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a9bde5b3a1b5,41785,-1] 2024-11-15T17:00:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:00:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:33,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:00:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8dec09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:00:33,009 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,009 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,011 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,020 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,021 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,021 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,021 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,023 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,024 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:33,024 DEBUG [Thread-2297 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,024 DEBUG [Thread-2297 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-15T17:00:33,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:00:33,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,028 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:33,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,029 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:33,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,036 DEBUG [Thread-2300 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:33,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,044 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:33,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:00:33,046 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:00:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=-1] 2024-11-15T17:00:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:00:33,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:00:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5] 2024-11-15T17:00:33,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690093070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690093075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690093078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c7da72964ca438383dac56c742a71ce_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690033039/Put/seqid=0 2024-11-15T17:00:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690093081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690093092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742335_1511 (size=12154) 2024-11-15T17:00:33,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:33,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c7da72964ca438383dac56c742a71ce_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c7da72964ca438383dac56c742a71ce_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:00:33,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2d13a2d653434761a0735494dfa52bfd, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:33,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2d13a2d653434761a0735494dfa52bfd is 175, key is test_row_0/A:col10/1731690033039/Put/seqid=0 2024-11-15T17:00:33,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:33,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690093186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690093186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690093196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690093196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742336_1512 (size=30955) 2024-11-15T17:00:33,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690093206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:00:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690093396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690093396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690093406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690093406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690093416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,602 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2d13a2d653434761a0735494dfa52bfd 2024-11-15T17:00:33,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:33,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:33,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/1818bafadbd54ef6a25a85c2f7ade75c is 50, key is test_row_0/B:col10/1731690033039/Put/seqid=0 2024-11-15T17:00:33,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:00:33,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742337_1513 (size=12001) 2024-11-15T17:00:33,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690093706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690093706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690093716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690093716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690093725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:33,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:33,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:33,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/1818bafadbd54ef6a25a85c2f7ade75c 2024-11-15T17:00:34,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/4e2dad1e87e141c3b8b7da92db7bd642 is 50, key is test_row_0/C:col10/1731690033039/Put/seqid=0 2024-11-15T17:00:34,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:00:34,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742338_1514 (size=12001) 2024-11-15T17:00:34,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:34,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690094216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690094216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:34,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:34,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690094226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690094226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690094236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:34,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:34,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:34,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:34,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:34,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:34,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:34,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/4e2dad1e87e141c3b8b7da92db7bd642 2024-11-15T17:00:34,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2d13a2d653434761a0735494dfa52bfd as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2d13a2d653434761a0735494dfa52bfd 2024-11-15T17:00:34,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2d13a2d653434761a0735494dfa52bfd, entries=150, sequenceid=19, filesize=30.2 K 2024-11-15T17:00:34,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/1818bafadbd54ef6a25a85c2f7ade75c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1818bafadbd54ef6a25a85c2f7ade75c 2024-11-15T17:00:34,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1818bafadbd54ef6a25a85c2f7ade75c, entries=150, sequenceid=19, filesize=11.7 K 2024-11-15T17:00:34,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/4e2dad1e87e141c3b8b7da92db7bd642 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/4e2dad1e87e141c3b8b7da92db7bd642 2024-11-15T17:00:34,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/4e2dad1e87e141c3b8b7da92db7bd642, entries=150, sequenceid=19, filesize=11.7 K 2024-11-15T17:00:34,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 696d98f18be792cdc1435b3b0e2fd21e in 1547ms, sequenceid=19, compaction requested=false 2024-11-15T17:00:34,591 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T17:00:34,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:34,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:00:34,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,716 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T17:00:34,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155499068c441d456e9c6ce0121e8dada9_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690033075/Put/seqid=0 2024-11-15T17:00:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742339_1515 (size=12154) 2024-11-15T17:00:34,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,808 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155499068c441d456e9c6ce0121e8dada9_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155499068c441d456e9c6ce0121e8dada9_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:34,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/f1b412573e2549e78e08d43b6b021a4d, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:34,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/f1b412573e2549e78e08d43b6b021a4d is 175, key is test_row_0/A:col10/1731690033075/Put/seqid=0 2024-11-15T17:00:34,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742340_1516 (size=30955) 2024-11-15T17:00:34,828 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/f1b412573e2549e78e08d43b6b021a4d 2024-11-15T17:00:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/43747ef607834f7790a6d0ac3aa71fe8 is 50, key is test_row_0/B:col10/1731690033075/Put/seqid=0 2024-11-15T17:00:34,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742341_1517 (size=12001) 2024-11-15T17:00:34,842 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/43747ef607834f7790a6d0ac3aa71fe8 2024-11-15T17:00:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/e6fc65ce550a47a28ef03431235933c1 is 50, key is test_row_0/C:col10/1731690033075/Put/seqid=0 2024-11-15T17:00:34,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742342_1518 (size=12001) 2024-11-15T17:00:34,858 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/e6fc65ce550a47a28ef03431235933c1 2024-11-15T17:00:34,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/f1b412573e2549e78e08d43b6b021a4d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f1b412573e2549e78e08d43b6b021a4d 2024-11-15T17:00:34,868 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f1b412573e2549e78e08d43b6b021a4d, entries=150, sequenceid=41, filesize=30.2 K 2024-11-15T17:00:34,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/43747ef607834f7790a6d0ac3aa71fe8 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/43747ef607834f7790a6d0ac3aa71fe8 2024-11-15T17:00:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,872 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/43747ef607834f7790a6d0ac3aa71fe8, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T17:00:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/e6fc65ce550a47a28ef03431235933c1 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e6fc65ce550a47a28ef03431235933c1 2024-11-15T17:00:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,877 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e6fc65ce550a47a28ef03431235933c1, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T17:00:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 696d98f18be792cdc1435b3b0e2fd21e in 162ms, sequenceid=41, compaction requested=false 2024-11-15T17:00:34,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:34,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:34,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-15T17:00:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-15T17:00:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T17:00:34,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-15T17:00:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8580 sec 2024-11-15T17:00:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:00:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,186 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-15T17:00:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:00:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,189 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,190 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:35,277 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:35,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:00:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b481222c14e344cca54cf869176b79cb_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690035275/Put/seqid=0 2024-11-15T17:00:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:00:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742343_1519 (size=19474) 2024-11-15T17:00:35,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b481222c14e344cca54cf869176b79cb_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b481222c14e344cca54cf869176b79cb_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,315 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/c9235298d8554643b8915f97b77a77dc, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/c9235298d8554643b8915f97b77a77dc is 175, key is test_row_0/A:col10/1731690035275/Put/seqid=0 2024-11-15T17:00:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:35,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742344_1520 (size=56733) 2024-11-15T17:00:35,321 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/c9235298d8554643b8915f97b77a77dc 2024-11-15T17:00:35,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/c3211f5378e34fc7b87f7e554eda3eb5 is 50, key is test_row_0/B:col10/1731690035275/Put/seqid=0 2024-11-15T17:00:35,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742345_1521 (size=12001) 2024-11-15T17:00:35,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/c3211f5378e34fc7b87f7e554eda3eb5 2024-11-15T17:00:35,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690095327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690095328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690095329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690095331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690095335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/84f87c62e8a948408707585b3086e61b is 50, key is test_row_0/C:col10/1731690035275/Put/seqid=0 2024-11-15T17:00:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:00:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:35,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742346_1522 (size=12001) 2024-11-15T17:00:35,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690095445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690095445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690095446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690095446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690095446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:00:35,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:35,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:00:35,610 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:00:35,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:00:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690095655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690095656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690095656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690095656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690095656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/84f87c62e8a948408707585b3086e61b 2024-11-15T17:00:35,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/c9235298d8554643b8915f97b77a77dc as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/c9235298d8554643b8915f97b77a77dc 2024-11-15T17:00:35,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/c9235298d8554643b8915f97b77a77dc, entries=300, sequenceid=52, filesize=55.4 K 2024-11-15T17:00:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/c3211f5378e34fc7b87f7e554eda3eb5 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c3211f5378e34fc7b87f7e554eda3eb5 2024-11-15T17:00:35,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c3211f5378e34fc7b87f7e554eda3eb5, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T17:00:35,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/84f87c62e8a948408707585b3086e61b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/84f87c62e8a948408707585b3086e61b 2024-11-15T17:00:35,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/84f87c62e8a948408707585b3086e61b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T17:00:35,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 696d98f18be792cdc1435b3b0e2fd21e in 496ms, sequenceid=52, compaction requested=true 2024-11-15T17:00:35,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:35,774 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:35,774 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:35,774 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:35,774 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:35,775 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,775 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2d13a2d653434761a0735494dfa52bfd, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f1b412573e2549e78e08d43b6b021a4d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/c9235298d8554643b8915f97b77a77dc] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=115.9 K 2024-11-15T17:00:35,775 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,775 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2d13a2d653434761a0735494dfa52bfd, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f1b412573e2549e78e08d43b6b021a4d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/c9235298d8554643b8915f97b77a77dc] 2024-11-15T17:00:35,775 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:35,775 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:35,775 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,775 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1818bafadbd54ef6a25a85c2f7ade75c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/43747ef607834f7790a6d0ac3aa71fe8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c3211f5378e34fc7b87f7e554eda3eb5] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.2 K 2024-11-15T17:00:35,776 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1818bafadbd54ef6a25a85c2f7ade75c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731690033039 2024-11-15T17:00:35,776 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d13a2d653434761a0735494dfa52bfd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731690033039 2024-11-15T17:00:35,776 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b412573e2549e78e08d43b6b021a4d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731690033067 2024-11-15T17:00:35,777 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 43747ef607834f7790a6d0ac3aa71fe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731690033067 2024-11-15T17:00:35,777 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9235298d8554643b8915f97b77a77dc, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731690035247 2024-11-15T17:00:35,777 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting c3211f5378e34fc7b87f7e554eda3eb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731690035247 2024-11-15T17:00:35,782 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:35,786 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c552c22819154ca0a7cbf2cb4f6f47c5_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:35,788 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:35,788 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c552c22819154ca0a7cbf2cb4f6f47c5_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:35,789 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c552c22819154ca0a7cbf2cb4f6f47c5_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:35,789 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/d6cf0dac506743faa707fdaac182966a is 50, key is test_row_0/B:col10/1731690035275/Put/seqid=0 2024-11-15T17:00:35,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742347_1523 (size=4469) 2024-11-15T17:00:35,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742348_1524 (size=12104) 2024-11-15T17:00:35,799 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#444 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:35,799 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/59e81cc4516c49f097ed5cfb2be76516 is 175, key is test_row_0/A:col10/1731690035275/Put/seqid=0 2024-11-15T17:00:35,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:00:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,804 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:00:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4387e6cc8b642b0a9e70cf9e3191b38_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690035326/Put/seqid=0 2024-11-15T17:00:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:00:35,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742349_1525 (size=31058) 2024-11-15T17:00:35,824 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/59e81cc4516c49f097ed5cfb2be76516 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/59e81cc4516c49f097ed5cfb2be76516 2024-11-15T17:00:35,831 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into 59e81cc4516c49f097ed5cfb2be76516(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:35,831 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:35,831 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=13, startTime=1731690035773; duration=0sec 2024-11-15T17:00:35,831 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:35,831 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:35,831 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742350_1526 (size=12154) 2024-11-15T17:00:35,832 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:35,832 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:35,833 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:35,833 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/4e2dad1e87e141c3b8b7da92db7bd642, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e6fc65ce550a47a28ef03431235933c1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/84f87c62e8a948408707585b3086e61b] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.2 K 2024-11-15T17:00:35,833 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e2dad1e87e141c3b8b7da92db7bd642, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731690033039 2024-11-15T17:00:35,833 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6fc65ce550a47a28ef03431235933c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731690033067 2024-11-15T17:00:35,834 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84f87c62e8a948408707585b3086e61b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731690035247 2024-11-15T17:00:35,843 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:35,844 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8c008bea96df4709b1f0d355dea85f6e is 50, key is test_row_0/C:col10/1731690035275/Put/seqid=0 2024-11-15T17:00:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742351_1527 (size=12104) 2024-11-15T17:00:35,859 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8c008bea96df4709b1f0d355dea85f6e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8c008bea96df4709b1f0d355dea85f6e 2024-11-15T17:00:35,867 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into 8c008bea96df4709b1f0d355dea85f6e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:35,867 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:35,867 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=13, startTime=1731690035774; duration=0sec 2024-11-15T17:00:35,867 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:35,867 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:00:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:35,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:35,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690095972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690095977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690095977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690095978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690095979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690096086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690096086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690096086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690096086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690096086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,206 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/d6cf0dac506743faa707fdaac182966a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/d6cf0dac506743faa707fdaac182966a 2024-11-15T17:00:36,213 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into d6cf0dac506743faa707fdaac182966a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:36,213 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:36,213 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=13, startTime=1731690035774; duration=0sec 2024-11-15T17:00:36,213 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:36,213 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:36,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:36,237 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4387e6cc8b642b0a9e70cf9e3191b38_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4387e6cc8b642b0a9e70cf9e3191b38_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/27a8d8f06f874d77a032fa66f7603147, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:36,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/27a8d8f06f874d77a032fa66f7603147 is 175, key is test_row_0/A:col10/1731690035326/Put/seqid=0 2024-11-15T17:00:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742352_1528 (size=30955) 2024-11-15T17:00:36,271 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/27a8d8f06f874d77a032fa66f7603147 2024-11-15T17:00:36,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/1fb9c47a9f1942deb7bc3183ea73e092 is 50, key is test_row_0/B:col10/1731690035326/Put/seqid=0 2024-11-15T17:00:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690096296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690096296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690096296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690096296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690096296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:00:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742353_1529 (size=12001) 2024-11-15T17:00:36,334 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/1fb9c47a9f1942deb7bc3183ea73e092 2024-11-15T17:00:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/019e1647642c423690d77db16b248372 is 50, key is test_row_0/C:col10/1731690035326/Put/seqid=0 2024-11-15T17:00:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742354_1530 (size=12001) 2024-11-15T17:00:36,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690096605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690096605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690096616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690096616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690096617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:36,776 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/019e1647642c423690d77db16b248372 2024-11-15T17:00:36,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/27a8d8f06f874d77a032fa66f7603147 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/27a8d8f06f874d77a032fa66f7603147 2024-11-15T17:00:36,784 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/27a8d8f06f874d77a032fa66f7603147, entries=150, sequenceid=77, filesize=30.2 K 2024-11-15T17:00:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/1fb9c47a9f1942deb7bc3183ea73e092 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1fb9c47a9f1942deb7bc3183ea73e092 2024-11-15T17:00:36,788 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1fb9c47a9f1942deb7bc3183ea73e092, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T17:00:36,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/019e1647642c423690d77db16b248372 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/019e1647642c423690d77db16b248372 2024-11-15T17:00:36,792 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/019e1647642c423690d77db16b248372, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T17:00:36,793 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 696d98f18be792cdc1435b3b0e2fd21e in 989ms, sequenceid=77, compaction requested=false 2024-11-15T17:00:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-15T17:00:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-15T17:00:36,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T17:00:36,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-11-15T17:00:36,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.6120 sec 2024-11-15T17:00:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:37,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:37,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:00:37,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111571163ca99b9e451f8cffc001f16445e3_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690037116/Put/seqid=0 2024-11-15T17:00:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690097151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690097152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690097153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690097153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690097153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742355_1531 (size=14594) 2024-11-15T17:00:37,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690097266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690097266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690097266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690097266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690097266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:00:37,336 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:37,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-15T17:00:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:00:37,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:37,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:37,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:00:37,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690097476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690097476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690097476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690097476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690097476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:00:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,565 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:37,568 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111571163ca99b9e451f8cffc001f16445e3_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111571163ca99b9e451f8cffc001f16445e3_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:37,569 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/1438cc9bd03f47d0a70f03a24392b328, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:37,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/1438cc9bd03f47d0a70f03a24392b328 is 175, key is test_row_0/A:col10/1731690037116/Put/seqid=0 2024-11-15T17:00:37,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742356_1532 (size=39549) 2024-11-15T17:00:37,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:00:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:00:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690097786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690097786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690097786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690097786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690097787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:37,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:00:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:00:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:37,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:00:37,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/1438cc9bd03f47d0a70f03a24392b328 2024-11-15T17:00:37,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/9107154f7a87480db09bc129d0dea7ee is 50, key is test_row_0/B:col10/1731690037116/Put/seqid=0 2024-11-15T17:00:37,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742357_1533 (size=12001) 2024-11-15T17:00:37,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/9107154f7a87480db09bc129d0dea7ee 2024-11-15T17:00:37,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/2c8828de633a428abe2fe79b7741efb0 is 50, key is test_row_0/C:col10/1731690037116/Put/seqid=0 2024-11-15T17:00:38,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742358_1534 (size=12001) 2024-11-15T17:00:38,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/2c8828de633a428abe2fe79b7741efb0 2024-11-15T17:00:38,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/1438cc9bd03f47d0a70f03a24392b328 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/1438cc9bd03f47d0a70f03a24392b328 2024-11-15T17:00:38,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/1438cc9bd03f47d0a70f03a24392b328, entries=200, sequenceid=94, filesize=38.6 K 2024-11-15T17:00:38,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/9107154f7a87480db09bc129d0dea7ee as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/9107154f7a87480db09bc129d0dea7ee 2024-11-15T17:00:38,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/9107154f7a87480db09bc129d0dea7ee, entries=150, sequenceid=94, filesize=11.7 K 2024-11-15T17:00:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/2c8828de633a428abe2fe79b7741efb0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2c8828de633a428abe2fe79b7741efb0 2024-11-15T17:00:38,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2c8828de633a428abe2fe79b7741efb0, entries=150, sequenceid=94, filesize=11.7 K 2024-11-15T17:00:38,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 696d98f18be792cdc1435b3b0e2fd21e in 908ms, sequenceid=94, compaction requested=true 2024-11-15T17:00:38,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:38,026 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:38,027 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:38,027 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:38,028 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:38,028 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:38,028 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:38,028 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:38,028 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/d6cf0dac506743faa707fdaac182966a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1fb9c47a9f1942deb7bc3183ea73e092, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/9107154f7a87480db09bc129d0dea7ee] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.3 K 2024-11-15T17:00:38,028 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:38,028 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/59e81cc4516c49f097ed5cfb2be76516, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/27a8d8f06f874d77a032fa66f7603147, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/1438cc9bd03f47d0a70f03a24392b328] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=99.2 K 2024-11-15T17:00:38,028 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:38,028 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/59e81cc4516c49f097ed5cfb2be76516, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/27a8d8f06f874d77a032fa66f7603147, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/1438cc9bd03f47d0a70f03a24392b328] 2024-11-15T17:00:38,028 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting d6cf0dac506743faa707fdaac182966a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731690035247 2024-11-15T17:00:38,028 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e81cc4516c49f097ed5cfb2be76516, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731690035247 2024-11-15T17:00:38,028 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb9c47a9f1942deb7bc3183ea73e092, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731690035326 2024-11-15T17:00:38,029 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27a8d8f06f874d77a032fa66f7603147, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731690035326 2024-11-15T17:00:38,029 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 9107154f7a87480db09bc129d0dea7ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731690035976 2024-11-15T17:00:38,029 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1438cc9bd03f47d0a70f03a24392b328, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731690035976 2024-11-15T17:00:38,034 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:38,035 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:38,036 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/a3b529d703c3462d91f2cc1718fabf74 is 50, key is test_row_0/B:col10/1731690037116/Put/seqid=0 2024-11-15T17:00:38,042 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a0787af31d4d4a349c4360ef536543b4_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:38,044 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a0787af31d4d4a349c4360ef536543b4_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:38,045 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a0787af31d4d4a349c4360ef536543b4_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:38,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742359_1535 (size=12207) 2024-11-15T17:00:38,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742360_1536 (size=4469) 2024-11-15T17:00:38,055 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#454 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:38,056 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/6d2afbd03180476ebfbd70c6ce06ea7c is 175, key is test_row_0/A:col10/1731690037116/Put/seqid=0 2024-11-15T17:00:38,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742361_1537 (size=31161) 2024-11-15T17:00:38,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:00:38,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:38,104 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:00:38,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f72187573011466a98f9c9bf91da1465_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690037151/Put/seqid=0 2024-11-15T17:00:38,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742362_1538 (size=12154) 2024-11-15T17:00:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:38,152 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f72187573011466a98f9c9bf91da1465_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f72187573011466a98f9c9bf91da1465_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/faa2d7ee2a314d71b8c9cc5eb7df0563, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:38,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/faa2d7ee2a314d71b8c9cc5eb7df0563 is 175, key is test_row_0/A:col10/1731690037151/Put/seqid=0 2024-11-15T17:00:38,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742363_1539 (size=30955) 2024-11-15T17:00:38,171 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/faa2d7ee2a314d71b8c9cc5eb7df0563 2024-11-15T17:00:38,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/c2e1d3469f4f48f79a0eeeaa352b809b is 50, key is test_row_0/B:col10/1731690037151/Put/seqid=0 2024-11-15T17:00:38,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742364_1540 (size=12001) 2024-11-15T17:00:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:38,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:38,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690098307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690098309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690098310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690098311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690098311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690098415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690098426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690098426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690098426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690098426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,458 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/a3b529d703c3462d91f2cc1718fabf74 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a3b529d703c3462d91f2cc1718fabf74 2024-11-15T17:00:38,464 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into a3b529d703c3462d91f2cc1718fabf74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:38,464 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:38,464 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=13, startTime=1731690038027; duration=0sec 2024-11-15T17:00:38,464 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:38,464 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:38,464 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:38,465 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:38,465 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:38,465 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:38,465 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8c008bea96df4709b1f0d355dea85f6e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/019e1647642c423690d77db16b248372, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2c8828de633a428abe2fe79b7741efb0] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.3 K 2024-11-15T17:00:38,466 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c008bea96df4709b1f0d355dea85f6e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731690035247 2024-11-15T17:00:38,466 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 019e1647642c423690d77db16b248372, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731690035326 2024-11-15T17:00:38,467 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c8828de633a428abe2fe79b7741efb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731690035976 2024-11-15T17:00:38,470 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/6d2afbd03180476ebfbd70c6ce06ea7c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/6d2afbd03180476ebfbd70c6ce06ea7c 2024-11-15T17:00:38,476 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into 6d2afbd03180476ebfbd70c6ce06ea7c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:00:38,476 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:38,476 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=13, startTime=1731690038026; duration=0sec 2024-11-15T17:00:38,476 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:38,476 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:38,478 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:38,479 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a55e6c9593254941bca99d7a5cb03b2c is 50, key is test_row_0/C:col10/1731690037116/Put/seqid=0 2024-11-15T17:00:38,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742365_1541 (size=12207) 2024-11-15T17:00:38,509 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a55e6c9593254941bca99d7a5cb03b2c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a55e6c9593254941bca99d7a5cb03b2c 2024-11-15T17:00:38,515 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into a55e6c9593254941bca99d7a5cb03b2c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:38,515 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:38,515 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=13, startTime=1731690038027; duration=0sec 2024-11-15T17:00:38,515 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:38,515 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:00:38,623 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/c2e1d3469f4f48f79a0eeeaa352b809b 2024-11-15T17:00:38,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690098626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/5d069544bd8046ef9b7e077fd9840523 is 50, key is test_row_0/C:col10/1731690037151/Put/seqid=0 2024-11-15T17:00:38,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690098636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690098636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690098636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690098637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742366_1542 (size=12001) 2024-11-15T17:00:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690098936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690098945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690098946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690098946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690098946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,066 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/5d069544bd8046ef9b7e077fd9840523 2024-11-15T17:00:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/faa2d7ee2a314d71b8c9cc5eb7df0563 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/faa2d7ee2a314d71b8c9cc5eb7df0563 2024-11-15T17:00:39,074 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/faa2d7ee2a314d71b8c9cc5eb7df0563, entries=150, sequenceid=117, filesize=30.2 K 2024-11-15T17:00:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/c2e1d3469f4f48f79a0eeeaa352b809b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c2e1d3469f4f48f79a0eeeaa352b809b 2024-11-15T17:00:39,077 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c2e1d3469f4f48f79a0eeeaa352b809b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T17:00:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/5d069544bd8046ef9b7e077fd9840523 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d069544bd8046ef9b7e077fd9840523 2024-11-15T17:00:39,081 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d069544bd8046ef9b7e077fd9840523, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T17:00:39,085 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 696d98f18be792cdc1435b3b0e2fd21e in 980ms, sequenceid=117, compaction requested=false 2024-11-15T17:00:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-15T17:00:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-15T17:00:39,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-15T17:00:39,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7490 sec 2024-11-15T17:00:39,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.7540 sec 2024-11-15T17:00:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:39,447 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:39,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:00:39,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111584430137c3b140d89e9107bb90df1ff4_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690038301/Put/seqid=0 2024-11-15T17:00:39,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742367_1543 (size=14694) 2024-11-15T17:00:39,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690099476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690099476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690099477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690099480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690099482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:00:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,486 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:39,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-15T17:00:39,488 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:00:39,489 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:39,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690099585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690099586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690099586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:00:39,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690099597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690099597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:39,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:39,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:39,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:39,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:39,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:00:39,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690099806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:39,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690099806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:39,858 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:39,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111584430137c3b140d89e9107bb90df1ff4_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584430137c3b140d89e9107bb90df1ff4_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:39,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/659cec8db1e6422fa5bb31cb9d24364d, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:39,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/659cec8db1e6422fa5bb31cb9d24364d is 175, key is test_row_0/A:col10/1731690038301/Put/seqid=0 2024-11-15T17:00:39,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742368_1544 (size=39649) 2024-11-15T17:00:39,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:39,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690100106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690100106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690100106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:00:40,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690100116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690100116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:40,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:40,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/659cec8db1e6422fa5bb31cb9d24364d 2024-11-15T17:00:40,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/cab12cbc42944ced9f59bb724bea57a7 is 50, key is test_row_0/B:col10/1731690038301/Put/seqid=0 2024-11-15T17:00:40,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742369_1545 (size=12101) 2024-11-15T17:00:40,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:40,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:40,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690100616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690100616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690100616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:00:40,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690100627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690100627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:40,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/cab12cbc42944ced9f59bb724bea57a7 2024-11-15T17:00:40,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/7f940e943bc94408aaf5747663fbe6e4 is 50, key is test_row_0/C:col10/1731690038301/Put/seqid=0 2024-11-15T17:00:40,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742370_1546 (size=12101) 2024-11-15T17:00:40,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/7f940e943bc94408aaf5747663fbe6e4 2024-11-15T17:00:40,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/659cec8db1e6422fa5bb31cb9d24364d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/659cec8db1e6422fa5bb31cb9d24364d 2024-11-15T17:00:40,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/659cec8db1e6422fa5bb31cb9d24364d, entries=200, sequenceid=134, filesize=38.7 K 2024-11-15T17:00:40,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/cab12cbc42944ced9f59bb724bea57a7 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/cab12cbc42944ced9f59bb724bea57a7 2024-11-15T17:00:40,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/cab12cbc42944ced9f59bb724bea57a7, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T17:00:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/7f940e943bc94408aaf5747663fbe6e4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/7f940e943bc94408aaf5747663fbe6e4 2024-11-15T17:00:40,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/7f940e943bc94408aaf5747663fbe6e4, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T17:00:40,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 696d98f18be792cdc1435b3b0e2fd21e in 1266ms, sequenceid=134, compaction requested=true 2024-11-15T17:00:40,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:40,714 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:40,715 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:40,715 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:40,715 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,715 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/6d2afbd03180476ebfbd70c6ce06ea7c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/faa2d7ee2a314d71b8c9cc5eb7df0563, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/659cec8db1e6422fa5bb31cb9d24364d] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=99.4 K 2024-11-15T17:00:40,715 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,715 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/6d2afbd03180476ebfbd70c6ce06ea7c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/faa2d7ee2a314d71b8c9cc5eb7df0563, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/659cec8db1e6422fa5bb31cb9d24364d] 2024-11-15T17:00:40,716 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2afbd03180476ebfbd70c6ce06ea7c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731690035976 2024-11-15T17:00:40,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T17:00:40,716 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa2d7ee2a314d71b8c9cc5eb7df0563, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731690037149 2024-11-15T17:00:40,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,717 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T17:00:40,717 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 659cec8db1e6422fa5bb31cb9d24364d, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731690038301 2024-11-15T17:00:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:40,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:40,720 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:40,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:40,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:40,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:40,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:40,723 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:40,723 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:40,723 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,723 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a3b529d703c3462d91f2cc1718fabf74, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c2e1d3469f4f48f79a0eeeaa352b809b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/cab12cbc42944ced9f59bb724bea57a7] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.5 K 2024-11-15T17:00:40,724 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b529d703c3462d91f2cc1718fabf74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731690035976 2024-11-15T17:00:40,724 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting c2e1d3469f4f48f79a0eeeaa352b809b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731690037149 2024-11-15T17:00:40,725 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting cab12cbc42944ced9f59bb724bea57a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731690038301 2024-11-15T17:00:40,728 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:40,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c3b7de8e40949f8a6ecbcc2afd411c9_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690039479/Put/seqid=0 2024-11-15T17:00:40,748 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111593da9cb14ca74d2cb6e1234bdb7e3bca_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:40,750 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111593da9cb14ca74d2cb6e1234bdb7e3bca_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:40,750 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111593da9cb14ca74d2cb6e1234bdb7e3bca_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:40,753 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:40,753 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/4a86b351556b4be2a7ddb1def1d806db is 50, key is test_row_0/B:col10/1731690038301/Put/seqid=0 2024-11-15T17:00:40,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742371_1547 (size=12304) 2024-11-15T17:00:40,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742373_1549 (size=12409) 2024-11-15T17:00:40,808 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/4a86b351556b4be2a7ddb1def1d806db as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/4a86b351556b4be2a7ddb1def1d806db 2024-11-15T17:00:40,815 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into 4a86b351556b4be2a7ddb1def1d806db(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:40,815 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:40,815 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=13, startTime=1731690040720; duration=0sec 2024-11-15T17:00:40,815 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:40,815 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:40,815 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:40,817 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:40,817 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:40,817 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:40,817 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a55e6c9593254941bca99d7a5cb03b2c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d069544bd8046ef9b7e077fd9840523, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/7f940e943bc94408aaf5747663fbe6e4] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.5 K 2024-11-15T17:00:40,818 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a55e6c9593254941bca99d7a5cb03b2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731690035976 2024-11-15T17:00:40,818 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d069544bd8046ef9b7e077fd9840523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731690037149 2024-11-15T17:00:40,818 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f940e943bc94408aaf5747663fbe6e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731690038301 2024-11-15T17:00:40,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742372_1548 (size=4469) 2024-11-15T17:00:40,827 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:40,827 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/833ce57303b240c9bbf7d3fd56cbaac4 is 50, key is test_row_0/C:col10/1731690038301/Put/seqid=0 2024-11-15T17:00:40,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742374_1550 (size=12409) 2024-11-15T17:00:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:41,183 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c3b7de8e40949f8a6ecbcc2afd411c9_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c3b7de8e40949f8a6ecbcc2afd411c9_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/752b74df24fb454d8c133fce34386167, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/752b74df24fb454d8c133fce34386167 is 175, key is test_row_0/A:col10/1731690039479/Put/seqid=0 2024-11-15T17:00:41,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742375_1551 (size=31105) 2024-11-15T17:00:41,230 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#463 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:41,230 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/b3109914f1004441914299a74e7247ac is 175, key is test_row_0/A:col10/1731690038301/Put/seqid=0 2024-11-15T17:00:41,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742376_1552 (size=31363) 2024-11-15T17:00:41,238 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/b3109914f1004441914299a74e7247ac as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/b3109914f1004441914299a74e7247ac 2024-11-15T17:00:41,242 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into b3109914f1004441914299a74e7247ac(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:41,242 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:41,242 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=13, startTime=1731690040714; duration=0sec 2024-11-15T17:00:41,242 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:41,242 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:41,256 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/833ce57303b240c9bbf7d3fd56cbaac4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/833ce57303b240c9bbf7d3fd56cbaac4 2024-11-15T17:00:41,260 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into 833ce57303b240c9bbf7d3fd56cbaac4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:41,260 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:41,260 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=13, startTime=1731690040720; duration=0sec 2024-11-15T17:00:41,260 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:41,260 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:00:41,589 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/752b74df24fb454d8c133fce34386167 2024-11-15T17:00:41,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ca1b59b165a7408d80df691aeb255c31 is 50, key is test_row_0/B:col10/1731690039479/Put/seqid=0 2024-11-15T17:00:41,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742377_1553 (size=12151) 2024-11-15T17:00:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:41,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:00:41,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690101645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,650 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_2' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690101648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690101649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690101650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690101650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690101755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690101755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690101766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690101766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690101966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690101968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690101975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:41,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690101975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:41,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,000 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ca1b59b165a7408d80df691aeb255c31 2024-11-15T17:00:42,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/15e2856b0a494e98936a827c7e1b8931 is 50, key is test_row_0/C:col10/1731690039479/Put/seqid=0 2024-11-15T17:00:42,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742378_1554 (size=12151) 2024-11-15T17:00:42,020 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/15e2856b0a494e98936a827c7e1b8931 2024-11-15T17:00:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/752b74df24fb454d8c133fce34386167 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/752b74df24fb454d8c133fce34386167 2024-11-15T17:00:42,031 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/752b74df24fb454d8c133fce34386167, entries=150, sequenceid=156, filesize=30.4 K 2024-11-15T17:00:42,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ca1b59b165a7408d80df691aeb255c31 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ca1b59b165a7408d80df691aeb255c31 2024-11-15T17:00:42,038 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ca1b59b165a7408d80df691aeb255c31, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T17:00:42,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/15e2856b0a494e98936a827c7e1b8931 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/15e2856b0a494e98936a827c7e1b8931 2024-11-15T17:00:42,044 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/15e2856b0a494e98936a827c7e1b8931, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T17:00:42,045 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 696d98f18be792cdc1435b3b0e2fd21e in 1328ms, sequenceid=156, compaction requested=false 2024-11-15T17:00:42,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:42,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:42,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-15T17:00:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-15T17:00:42,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-15T17:00:42,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5580 sec 2024-11-15T17:00:42,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.5630 sec 2024-11-15T17:00:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:42,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:42,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:00:42,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111579323a253ca64ffc8d83e63647b921fc_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690042279/Put/seqid=0 2024-11-15T17:00:42,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742379_1555 (size=14794) 2024-11-15T17:00:42,302 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:42,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111579323a253ca64ffc8d83e63647b921fc_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579323a253ca64ffc8d83e63647b921fc_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:42,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/447a7d5625ee4ce29250daf9db666ad9, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:42,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/447a7d5625ee4ce29250daf9db666ad9 is 175, key is test_row_0/A:col10/1731690042279/Put/seqid=0 2024-11-15T17:00:42,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690102306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690102306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690102308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690102312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742380_1556 (size=39749) 2024-11-15T17:00:42,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690102416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690102416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690102416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690102425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690102626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690102626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690102626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690102635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,719 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/447a7d5625ee4ce29250daf9db666ad9 2024-11-15T17:00:42,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/04f97c1a6b604d118df2be0f1bcaba4a is 50, key is test_row_0/B:col10/1731690042279/Put/seqid=0 2024-11-15T17:00:42,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742381_1557 (size=12151) 2024-11-15T17:00:42,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690102935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690102935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690102935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:42,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690102946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:42,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:42,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:43,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/04f97c1a6b604d118df2be0f1bcaba4a 2024-11-15T17:00:43,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/53065d2d5262461e841c01943c0363f0 is 50, key is test_row_0/C:col10/1731690042279/Put/seqid=0 2024-11-15T17:00:43,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742382_1558 (size=12151) 2024-11-15T17:00:43,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690103446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690103446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690103446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690103456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:43,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/53065d2d5262461e841c01943c0363f0 2024-11-15T17:00:43,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/447a7d5625ee4ce29250daf9db666ad9 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/447a7d5625ee4ce29250daf9db666ad9 2024-11-15T17:00:43,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/447a7d5625ee4ce29250daf9db666ad9, entries=200, sequenceid=177, filesize=38.8 K 2024-11-15T17:00:43,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/04f97c1a6b604d118df2be0f1bcaba4a as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/04f97c1a6b604d118df2be0f1bcaba4a 2024-11-15T17:00:43,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/04f97c1a6b604d118df2be0f1bcaba4a, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T17:00:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/53065d2d5262461e841c01943c0363f0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/53065d2d5262461e841c01943c0363f0 2024-11-15T17:00:43,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/53065d2d5262461e841c01943c0363f0, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T17:00:43,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 696d98f18be792cdc1435b3b0e2fd21e in 1280ms, sequenceid=177, compaction requested=true 2024-11-15T17:00:43,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:43,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:43,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:43,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:43,560 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:43,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:43,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:43,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:43,561 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:43,561 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:43,561 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:43,561 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:43,561 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:43,561 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,561 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,561 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/4a86b351556b4be2a7ddb1def1d806db, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ca1b59b165a7408d80df691aeb255c31, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/04f97c1a6b604d118df2be0f1bcaba4a] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.9 K 2024-11-15T17:00:43,561 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/b3109914f1004441914299a74e7247ac, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/752b74df24fb454d8c133fce34386167, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/447a7d5625ee4ce29250daf9db666ad9] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=99.8 K 2024-11-15T17:00:43,562 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,562 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/b3109914f1004441914299a74e7247ac, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/752b74df24fb454d8c133fce34386167, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/447a7d5625ee4ce29250daf9db666ad9] 2024-11-15T17:00:43,563 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3109914f1004441914299a74e7247ac, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731690038301 2024-11-15T17:00:43,563 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 752b74df24fb454d8c133fce34386167, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731690039475 2024-11-15T17:00:43,563 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a86b351556b4be2a7ddb1def1d806db, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731690038301 2024-11-15T17:00:43,563 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447a7d5625ee4ce29250daf9db666ad9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731690041637 2024-11-15T17:00:43,563 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1b59b165a7408d80df691aeb255c31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731690039475 2024-11-15T17:00:43,564 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 04f97c1a6b604d118df2be0f1bcaba4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731690041637 2024-11-15T17:00:43,569 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:43,570 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:43,570 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ab8e2cc07ada4cdd969f998e3772d0f2 is 50, key is test_row_0/B:col10/1731690042279/Put/seqid=0 2024-11-15T17:00:43,571 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111574f519818e9046a780f6f2e0c8552d83_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:43,574 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111574f519818e9046a780f6f2e0c8552d83_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:43,574 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111574f519818e9046a780f6f2e0c8552d83_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:43,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742383_1559 (size=12561) 2024-11-15T17:00:43,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742384_1560 (size=4469) 2024-11-15T17:00:43,583 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#472 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:43,583 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/66bf8b96ce0a43db9146463015434ea3 is 175, key is test_row_0/A:col10/1731690042279/Put/seqid=0 2024-11-15T17:00:43,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742385_1561 (size=31515) 2024-11-15T17:00:43,591 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/66bf8b96ce0a43db9146463015434ea3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/66bf8b96ce0a43db9146463015434ea3 2024-11-15T17:00:43,595 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into 66bf8b96ce0a43db9146463015434ea3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:43,595 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:43,595 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=13, startTime=1731690043560; duration=0sec 2024-11-15T17:00:43,595 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:43,595 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:43,596 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:43,598 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:43,598 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:43,598 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,598 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/833ce57303b240c9bbf7d3fd56cbaac4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/15e2856b0a494e98936a827c7e1b8931, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/53065d2d5262461e841c01943c0363f0] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=35.9 K 2024-11-15T17:00:43,603 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833ce57303b240c9bbf7d3fd56cbaac4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731690038301 2024-11-15T17:00:43,603 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e2856b0a494e98936a827c7e1b8931, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731690039475 2024-11-15T17:00:43,603 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53065d2d5262461e841c01943c0363f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731690041637 2024-11-15T17:00:43,614 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:43,614 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/e02583a951894f0388cda208fcc785fa is 50, key is test_row_0/C:col10/1731690042279/Put/seqid=0 2024-11-15T17:00:43,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742386_1562 (size=12561) 2024-11-15T17:00:43,637 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/e02583a951894f0388cda208fcc785fa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e02583a951894f0388cda208fcc785fa 2024-11-15T17:00:43,644 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into e02583a951894f0388cda208fcc785fa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:43,644 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:43,644 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=13, startTime=1731690043560; duration=0sec 2024-11-15T17:00:43,644 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:43,644 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:00:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:00:43,666 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:43,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-15T17:00:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T17:00:43,669 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:43,670 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:43,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:43,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T17:00:43,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151a18f0e0fe0046bdaf7d39fb7392cb6f_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690042306/Put/seqid=0 2024-11-15T17:00:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742387_1563 (size=14794) 2024-11-15T17:00:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690103764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T17:00:43,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T17:00:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:43,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690103875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:43,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T17:00:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:43,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:43,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:43,979 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ab8e2cc07ada4cdd969f998e3772d0f2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ab8e2cc07ada4cdd969f998e3772d0f2 2024-11-15T17:00:43,983 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into ab8e2cc07ada4cdd969f998e3772d0f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:43,983 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:43,983 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=13, startTime=1731690043560; duration=0sec 2024-11-15T17:00:43,983 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:43,983 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T17:00:44,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690104086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,111 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:44,115 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151a18f0e0fe0046bdaf7d39fb7392cb6f_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a18f0e0fe0046bdaf7d39fb7392cb6f_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:44,116 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ab94d18ebab46299822b99550dafd90, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:44,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ab94d18ebab46299822b99550dafd90 is 175, key is test_row_0/A:col10/1731690042306/Put/seqid=0 2024-11-15T17:00:44,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T17:00:44,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:44,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:44,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:44,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742388_1564 (size=39749) 2024-11-15T17:00:44,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ab94d18ebab46299822b99550dafd90 2024-11-15T17:00:44,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/076b976ce1f94e818d14cea497060928 is 50, key is test_row_0/B:col10/1731690042306/Put/seqid=0 2024-11-15T17:00:44,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742389_1565 (size=12151) 2024-11-15T17:00:44,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/076b976ce1f94e818d14cea497060928 2024-11-15T17:00:44,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a367f397e29b40e7a7d9d1eb538c8d66 is 50, key is test_row_0/C:col10/1731690042306/Put/seqid=0 2024-11-15T17:00:44,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742390_1566 (size=12151) 2024-11-15T17:00:44,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a367f397e29b40e7a7d9d1eb538c8d66 2024-11-15T17:00:44,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ab94d18ebab46299822b99550dafd90 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ab94d18ebab46299822b99550dafd90 2024-11-15T17:00:44,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ab94d18ebab46299822b99550dafd90, entries=200, sequenceid=198, filesize=38.8 K 2024-11-15T17:00:44,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/076b976ce1f94e818d14cea497060928 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/076b976ce1f94e818d14cea497060928 2024-11-15T17:00:44,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/076b976ce1f94e818d14cea497060928, entries=150, sequenceid=198, filesize=11.9 K 2024-11-15T17:00:44,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a367f397e29b40e7a7d9d1eb538c8d66 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a367f397e29b40e7a7d9d1eb538c8d66 2024-11-15T17:00:44,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a367f397e29b40e7a7d9d1eb538c8d66, entries=150, sequenceid=198, filesize=11.9 K 2024-11-15T17:00:44,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 696d98f18be792cdc1435b3b0e2fd21e in 523ms, sequenceid=198, compaction requested=false 2024-11-15T17:00:44,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:44,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T17:00:44,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:44,284 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:00:44,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bfe03a0fb2564d168c857605e54e53e9_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690043752/Put/seqid=0 2024-11-15T17:00:44,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742391_1567 (size=12304) 2024-11-15T17:00:44,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T17:00:44,298 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bfe03a0fb2564d168c857605e54e53e9_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfe03a0fb2564d168c857605e54e53e9_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ce9581c1504a44b293bda3bf88a26ca8, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:44,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ce9581c1504a44b293bda3bf88a26ca8 is 175, key is test_row_0/A:col10/1731690043752/Put/seqid=0 2024-11-15T17:00:44,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742392_1568 (size=31105) 2024-11-15T17:00:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690104456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690104456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,459 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:44,459 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690104458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690104459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690104465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,467 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690104565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690104565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,703 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ce9581c1504a44b293bda3bf88a26ca8 2024-11-15T17:00:44,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/a33cf4e15f084064851df0d7af05b886 is 50, key is test_row_0/B:col10/1731690043752/Put/seqid=0 2024-11-15T17:00:44,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742393_1569 (size=12151) 2024-11-15T17:00:44,723 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/a33cf4e15f084064851df0d7af05b886 2024-11-15T17:00:44,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/f9b60b0211d1454288f0e407a9e5d255 is 50, key is test_row_0/C:col10/1731690043752/Put/seqid=0 2024-11-15T17:00:44,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742394_1570 (size=12151) 2024-11-15T17:00:44,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690104776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690104776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T17:00:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690105085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690105086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,135 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/f9b60b0211d1454288f0e407a9e5d255 2024-11-15T17:00:45,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ce9581c1504a44b293bda3bf88a26ca8 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ce9581c1504a44b293bda3bf88a26ca8 2024-11-15T17:00:45,142 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ce9581c1504a44b293bda3bf88a26ca8, entries=150, sequenceid=216, filesize=30.4 K 2024-11-15T17:00:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/a33cf4e15f084064851df0d7af05b886 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a33cf4e15f084064851df0d7af05b886 2024-11-15T17:00:45,155 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a33cf4e15f084064851df0d7af05b886, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T17:00:45,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/f9b60b0211d1454288f0e407a9e5d255 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f9b60b0211d1454288f0e407a9e5d255 2024-11-15T17:00:45,159 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f9b60b0211d1454288f0e407a9e5d255, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T17:00:45,160 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 696d98f18be792cdc1435b3b0e2fd21e in 876ms, sequenceid=216, compaction requested=true 2024-11-15T17:00:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-15T17:00:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-15T17:00:45,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-15T17:00:45,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4910 sec 2024-11-15T17:00:45,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.4970 sec 2024-11-15T17:00:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:45,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:45,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:00:45,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111572353f280e48422d926168e48f711a56_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690044457/Put/seqid=0 2024-11-15T17:00:45,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742395_1571 (size=14794) 2024-11-15T17:00:45,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690105631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690105632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690105746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690105746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T17:00:45,816 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:45,816 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-15T17:00:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:00:45,818 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:45,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:45,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:00:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690105956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690105956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:45,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T17:00:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:45,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:46,014 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111572353f280e48422d926168e48f711a56_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111572353f280e48422d926168e48f711a56_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:46,015 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/5735dfb3eb3a4df4959d076325ad13ec, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:46,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/5735dfb3eb3a4df4959d076325ad13ec is 175, key is test_row_0/A:col10/1731690044457/Put/seqid=0 2024-11-15T17:00:46,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742396_1572 (size=39749) 2024-11-15T17:00:46,020 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/5735dfb3eb3a4df4959d076325ad13ec 2024-11-15T17:00:46,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/bbe9db1ae16b4723976990f8d30c0952 is 50, key is test_row_0/B:col10/1731690044457/Put/seqid=0 2024-11-15T17:00:46,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742397_1573 (size=12151) 2024-11-15T17:00:46,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T17:00:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:00:46,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690106265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:46,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690106265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:46,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T17:00:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T17:00:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:00:46,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/bbe9db1ae16b4723976990f8d30c0952 2024-11-15T17:00:46,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/fe64abed29e54f9db1fa96e250cea760 is 50, key is test_row_0/C:col10/1731690044457/Put/seqid=0 2024-11-15T17:00:46,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742398_1574 (size=12151) 2024-11-15T17:00:46,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690106476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:46,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690106476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:46,479 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,479 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690106485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:46,488 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:46,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T17:00:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T17:00:46,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:46,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690106776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690106776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:46,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:46,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/fe64abed29e54f9db1fa96e250cea760 2024-11-15T17:00:46,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/5735dfb3eb3a4df4959d076325ad13ec as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/5735dfb3eb3a4df4959d076325ad13ec 2024-11-15T17:00:46,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/5735dfb3eb3a4df4959d076325ad13ec, entries=200, sequenceid=237, filesize=38.8 K 2024-11-15T17:00:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/bbe9db1ae16b4723976990f8d30c0952 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/bbe9db1ae16b4723976990f8d30c0952 2024-11-15T17:00:46,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/bbe9db1ae16b4723976990f8d30c0952, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T17:00:46,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/fe64abed29e54f9db1fa96e250cea760 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe64abed29e54f9db1fa96e250cea760 2024-11-15T17:00:46,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe64abed29e54f9db1fa96e250cea760, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T17:00:46,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 696d98f18be792cdc1435b3b0e2fd21e in 1290ms, sequenceid=237, compaction requested=true 2024-11-15T17:00:46,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:46,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:46,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:46,888 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:46,888 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:46,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:46,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:46,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:46,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:46,889 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:46,890 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:46,890 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,890 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ab8e2cc07ada4cdd969f998e3772d0f2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/076b976ce1f94e818d14cea497060928, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a33cf4e15f084064851df0d7af05b886, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/bbe9db1ae16b4723976990f8d30c0952] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=47.9 K 2024-11-15T17:00:46,890 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:46,890 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:46,890 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,890 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/66bf8b96ce0a43db9146463015434ea3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ab94d18ebab46299822b99550dafd90, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ce9581c1504a44b293bda3bf88a26ca8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/5735dfb3eb3a4df4959d076325ad13ec] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=138.8 K 2024-11-15T17:00:46,890 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,890 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/66bf8b96ce0a43db9146463015434ea3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ab94d18ebab46299822b99550dafd90, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ce9581c1504a44b293bda3bf88a26ca8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/5735dfb3eb3a4df4959d076325ad13ec] 2024-11-15T17:00:46,891 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8e2cc07ada4cdd969f998e3772d0f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731690041637 2024-11-15T17:00:46,891 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66bf8b96ce0a43db9146463015434ea3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731690041637 2024-11-15T17:00:46,891 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 076b976ce1f94e818d14cea497060928, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731690042305 2024-11-15T17:00:46,892 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab94d18ebab46299822b99550dafd90, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731690042305 2024-11-15T17:00:46,892 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a33cf4e15f084064851df0d7af05b886, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731690043727 2024-11-15T17:00:46,892 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce9581c1504a44b293bda3bf88a26ca8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731690043727 2024-11-15T17:00:46,892 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe9db1ae16b4723976990f8d30c0952, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731690044456 2024-11-15T17:00:46,893 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5735dfb3eb3a4df4959d076325ad13ec, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731690044456 2024-11-15T17:00:46,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T17:00:46,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:46,893 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T17:00:46,902 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:46,903 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/30bb1ed100224a61ad0fdf867e7a953b is 50, key is test_row_0/B:col10/1731690044457/Put/seqid=0 2024-11-15T17:00:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e65e823357d2473c830466e3da357e1e_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690045602/Put/seqid=0 2024-11-15T17:00:46,905 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:46,910 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ec250492d84b42518e87105c0b9aae1b_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:46,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742399_1575 (size=12697) 2024-11-15T17:00:46,913 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ec250492d84b42518e87105c0b9aae1b_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:46,913 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec250492d84b42518e87105c0b9aae1b_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:46,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742400_1576 (size=12304) 2024-11-15T17:00:46,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:46,926 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e65e823357d2473c830466e3da357e1e_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e65e823357d2473c830466e3da357e1e_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:46,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742401_1577 (size=4469) 2024-11-15T17:00:46,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/fe0021ee9b484bf4aa0305448bad6582, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/fe0021ee9b484bf4aa0305448bad6582 is 175, key is test_row_0/A:col10/1731690045602/Put/seqid=0 2024-11-15T17:00:46,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742402_1578 (size=31105) 2024-11-15T17:00:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:00:47,317 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/30bb1ed100224a61ad0fdf867e7a953b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/30bb1ed100224a61ad0fdf867e7a953b 2024-11-15T17:00:47,326 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into 30bb1ed100224a61ad0fdf867e7a953b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:47,326 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:47,326 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=12, startTime=1731690046888; duration=0sec 2024-11-15T17:00:47,326 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:47,326 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:47,326 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:00:47,328 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:00:47,328 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:47,329 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:47,329 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e02583a951894f0388cda208fcc785fa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a367f397e29b40e7a7d9d1eb538c8d66, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f9b60b0211d1454288f0e407a9e5d255, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe64abed29e54f9db1fa96e250cea760] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=47.9 K 2024-11-15T17:00:47,329 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#485 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:47,330 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/0b348c6a9c1840d4b84749abaa59f315 is 175, key is test_row_0/A:col10/1731690044457/Put/seqid=0 2024-11-15T17:00:47,330 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting e02583a951894f0388cda208fcc785fa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731690041637 2024-11-15T17:00:47,330 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a367f397e29b40e7a7d9d1eb538c8d66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731690042305 2024-11-15T17:00:47,331 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b60b0211d1454288f0e407a9e5d255, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731690043727 2024-11-15T17:00:47,332 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting fe64abed29e54f9db1fa96e250cea760, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731690044456 2024-11-15T17:00:47,344 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:47,345 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/6d13818e3b89420cb7cfba8e1b4fe00c is 50, key is test_row_0/C:col10/1731690044457/Put/seqid=0 2024-11-15T17:00:47,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742403_1579 (size=31651) 2024-11-15T17:00:47,349 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/fe0021ee9b484bf4aa0305448bad6582 2024-11-15T17:00:47,355 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/0b348c6a9c1840d4b84749abaa59f315 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/0b348c6a9c1840d4b84749abaa59f315 2024-11-15T17:00:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/3c9f01c2b8b84b21a1fa93818aaff168 is 50, key is test_row_0/B:col10/1731690045602/Put/seqid=0 2024-11-15T17:00:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742404_1580 (size=12697) 2024-11-15T17:00:47,363 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into 0b348c6a9c1840d4b84749abaa59f315(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:47,363 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:47,363 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=12, startTime=1731690046888; duration=0sec 2024-11-15T17:00:47,363 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:47,363 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:47,367 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/6d13818e3b89420cb7cfba8e1b4fe00c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/6d13818e3b89420cb7cfba8e1b4fe00c 2024-11-15T17:00:47,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742405_1581 (size=12151) 2024-11-15T17:00:47,372 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into 6d13818e3b89420cb7cfba8e1b4fe00c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:47,372 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:47,372 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=12, startTime=1731690046888; duration=0sec 2024-11-15T17:00:47,372 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:47,372 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:00:47,769 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/3c9f01c2b8b84b21a1fa93818aaff168 2024-11-15T17:00:47,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8fbd9bd85a1048e3a6f6aa8b1c43f017 is 50, key is test_row_0/C:col10/1731690045602/Put/seqid=0 2024-11-15T17:00:47,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742406_1582 (size=12151) 2024-11-15T17:00:47,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8fbd9bd85a1048e3a6f6aa8b1c43f017 2024-11-15T17:00:47,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/fe0021ee9b484bf4aa0305448bad6582 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/fe0021ee9b484bf4aa0305448bad6582 2024-11-15T17:00:47,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:47,789 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/fe0021ee9b484bf4aa0305448bad6582, entries=150, sequenceid=253, filesize=30.4 K 2024-11-15T17:00:47,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/3c9f01c2b8b84b21a1fa93818aaff168 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/3c9f01c2b8b84b21a1fa93818aaff168 2024-11-15T17:00:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,793 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/3c9f01c2b8b84b21a1fa93818aaff168, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T17:00:47,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8fbd9bd85a1048e3a6f6aa8b1c43f017 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8fbd9bd85a1048e3a6f6aa8b1c43f017 2024-11-15T17:00:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,796 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8fbd9bd85a1048e3a6f6aa8b1c43f017, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T17:00:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,797 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=20.13 KB/20610 for 696d98f18be792cdc1435b3b0e2fd21e in 904ms, sequenceid=253, compaction requested=false 2024-11-15T17:00:47,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:47,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:47,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-15T17:00:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-15T17:00:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-15T17:00:47,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9790 sec 2024-11-15T17:00:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.9850 sec 2024-11-15T17:00:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:00:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111596e820c14cd746899681245cb61e0d6c_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690047805/Put/seqid=0 2024-11-15T17:00:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:47,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742408_1584 (size=25158) 2024-11-15T17:00:47,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690107871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:47,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:47,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690107873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:00:47,966 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:47,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-15T17:00:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T17:00:47,970 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:47,970 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:47,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:47,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690107985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:47,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690107985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:47,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T17:00:48,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T17:00:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690108196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690108196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:48,254 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:48,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111596e820c14cd746899681245cb61e0d6c_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111596e820c14cd746899681245cb61e0d6c_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:48,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ef13dc2d85ab4109b342af0bb32e0962, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:48,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ef13dc2d85ab4109b342af0bb32e0962 is 175, key is test_row_0/A:col10/1731690047805/Put/seqid=0 2024-11-15T17:00:48,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742407_1583 (size=74791) 2024-11-15T17:00:48,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ef13dc2d85ab4109b342af0bb32e0962 2024-11-15T17:00:48,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T17:00:48,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:48,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ae7fedadd4ff4dafba00919a50ab8799 is 50, key is test_row_0/B:col10/1731690047805/Put/seqid=0 2024-11-15T17:00:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742409_1585 (size=9857) 2024-11-15T17:00:48,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ae7fedadd4ff4dafba00919a50ab8799 2024-11-15T17:00:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T17:00:48,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/2906c374dd934bd7a0d5c3375a8cab06 is 50, key is test_row_0/C:col10/1731690047805/Put/seqid=0 2024-11-15T17:00:48,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742410_1586 (size=9857) 2024-11-15T17:00:48,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T17:00:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690108506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690108506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:48,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T17:00:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T17:00:48,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/2906c374dd934bd7a0d5c3375a8cab06 2024-11-15T17:00:48,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/ef13dc2d85ab4109b342af0bb32e0962 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ef13dc2d85ab4109b342af0bb32e0962 2024-11-15T17:00:48,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ef13dc2d85ab4109b342af0bb32e0962, entries=400, sequenceid=267, filesize=73.0 K 2024-11-15T17:00:48,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/ae7fedadd4ff4dafba00919a50ab8799 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ae7fedadd4ff4dafba00919a50ab8799 2024-11-15T17:00:48,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ae7fedadd4ff4dafba00919a50ab8799, entries=100, sequenceid=267, filesize=9.6 K 2024-11-15T17:00:48,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/2906c374dd934bd7a0d5c3375a8cab06 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2906c374dd934bd7a0d5c3375a8cab06 2024-11-15T17:00:48,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2906c374dd934bd7a0d5c3375a8cab06, entries=100, sequenceid=267, filesize=9.6 K 2024-11-15T17:00:48,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 696d98f18be792cdc1435b3b0e2fd21e in 912ms, sequenceid=267, compaction requested=true 2024-11-15T17:00:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:48,722 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:48,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:48,722 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:48,723 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:48,723 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:48,723 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:48,723 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:48,723 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,723 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,723 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/30bb1ed100224a61ad0fdf867e7a953b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/3c9f01c2b8b84b21a1fa93818aaff168, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ae7fedadd4ff4dafba00919a50ab8799] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=33.9 K 2024-11-15T17:00:48,723 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/0b348c6a9c1840d4b84749abaa59f315, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/fe0021ee9b484bf4aa0305448bad6582, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ef13dc2d85ab4109b342af0bb32e0962] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=134.3 K 2024-11-15T17:00:48,723 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,723 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/0b348c6a9c1840d4b84749abaa59f315, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/fe0021ee9b484bf4aa0305448bad6582, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ef13dc2d85ab4109b342af0bb32e0962] 2024-11-15T17:00:48,723 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 30bb1ed100224a61ad0fdf867e7a953b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731690044456 2024-11-15T17:00:48,723 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b348c6a9c1840d4b84749abaa59f315, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731690044456 2024-11-15T17:00:48,724 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0021ee9b484bf4aa0305448bad6582, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731690045602 2024-11-15T17:00:48,724 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9f01c2b8b84b21a1fa93818aaff168, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731690045602 2024-11-15T17:00:48,724 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef13dc2d85ab4109b342af0bb32e0962, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731690047786 2024-11-15T17:00:48,724 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting ae7fedadd4ff4dafba00919a50ab8799, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731690047805 2024-11-15T17:00:48,731 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:48,732 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#492 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:48,732 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/78308d5672a5486590c842ba7880317d is 50, key is test_row_0/B:col10/1731690047805/Put/seqid=0 2024-11-15T17:00:48,733 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b68bf113854140699dcf82416d7cca56_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:48,735 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b68bf113854140699dcf82416d7cca56_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:48,735 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b68bf113854140699dcf82416d7cca56_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:48,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742411_1587 (size=12899) 2024-11-15T17:00:48,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742412_1588 (size=4469) 2024-11-15T17:00:48,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T17:00:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:48,741 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:00:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d66c1ec09def437e9d60c14155747eb7_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690047866/Put/seqid=0 2024-11-15T17:00:48,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742413_1589 (size=12454) 2024-11-15T17:00:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:49,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690109034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690109037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T17:00:49,141 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:49,142 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/f297678a0b6f4fb7add93701fb7b5dbe is 175, key is test_row_0/A:col10/1731690047805/Put/seqid=0 2024-11-15T17:00:49,145 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/78308d5672a5486590c842ba7880317d as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/78308d5672a5486590c842ba7880317d 2024-11-15T17:00:49,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742414_1590 (size=31960) 2024-11-15T17:00:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690109145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690109146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,150 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into 78308d5672a5486590c842ba7880317d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:49,150 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:49,150 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=13, startTime=1731690048722; duration=0sec 2024-11-15T17:00:49,150 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:49,150 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:49,150 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:49,151 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:49,151 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:49,151 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:49,151 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/6d13818e3b89420cb7cfba8e1b4fe00c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8fbd9bd85a1048e3a6f6aa8b1c43f017, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2906c374dd934bd7a0d5c3375a8cab06] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=33.9 K 2024-11-15T17:00:49,152 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d13818e3b89420cb7cfba8e1b4fe00c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731690044456 2024-11-15T17:00:49,152 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fbd9bd85a1048e3a6f6aa8b1c43f017, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731690045602 2024-11-15T17:00:49,152 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 2906c374dd934bd7a0d5c3375a8cab06, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731690047805 2024-11-15T17:00:49,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:49,156 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d66c1ec09def437e9d60c14155747eb7_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d66c1ec09def437e9d60c14155747eb7_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/168b5a0d8be64f8b98f2b10d02703957, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/168b5a0d8be64f8b98f2b10d02703957 is 175, key is test_row_0/A:col10/1731690047866/Put/seqid=0 2024-11-15T17:00:49,159 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#495 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:49,160 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/0daa56d1c8e644869d85bc9c6819e798 is 50, key is test_row_0/C:col10/1731690047805/Put/seqid=0 2024-11-15T17:00:49,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742415_1591 (size=31255) 2024-11-15T17:00:49,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742416_1592 (size=12899) 2024-11-15T17:00:49,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690109356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690109356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,550 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/f297678a0b6f4fb7add93701fb7b5dbe as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f297678a0b6f4fb7add93701fb7b5dbe 2024-11-15T17:00:49,554 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into f297678a0b6f4fb7add93701fb7b5dbe(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:49,554 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:49,554 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=13, startTime=1731690048722; duration=0sec 2024-11-15T17:00:49,554 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:49,554 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:49,567 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/168b5a0d8be64f8b98f2b10d02703957 2024-11-15T17:00:49,571 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/0daa56d1c8e644869d85bc9c6819e798 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/0daa56d1c8e644869d85bc9c6819e798 2024-11-15T17:00:49,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/412f32efe57a40e6ad85a6e83c84bc1b is 50, key is test_row_0/B:col10/1731690047866/Put/seqid=0 2024-11-15T17:00:49,575 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into 0daa56d1c8e644869d85bc9c6819e798(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:49,575 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:49,575 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=13, startTime=1731690048722; duration=0sec 2024-11-15T17:00:49,575 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:49,575 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:00:49,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742417_1593 (size=12301) 2024-11-15T17:00:49,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/412f32efe57a40e6ad85a6e83c84bc1b 2024-11-15T17:00:49,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a95e6fe9cc1f4aa287ca072ae967fd74 is 50, key is test_row_0/C:col10/1731690047866/Put/seqid=0 2024-11-15T17:00:49,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742418_1594 (size=12301) 2024-11-15T17:00:49,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690109666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690109666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:49,990 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a95e6fe9cc1f4aa287ca072ae967fd74 2024-11-15T17:00:49,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/168b5a0d8be64f8b98f2b10d02703957 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/168b5a0d8be64f8b98f2b10d02703957 2024-11-15T17:00:49,998 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/168b5a0d8be64f8b98f2b10d02703957, entries=150, sequenceid=292, filesize=30.5 K 2024-11-15T17:00:49,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/412f32efe57a40e6ad85a6e83c84bc1b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/412f32efe57a40e6ad85a6e83c84bc1b 2024-11-15T17:00:50,002 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/412f32efe57a40e6ad85a6e83c84bc1b, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T17:00:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/a95e6fe9cc1f4aa287ca072ae967fd74 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a95e6fe9cc1f4aa287ca072ae967fd74 2024-11-15T17:00:50,005 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a95e6fe9cc1f4aa287ca072ae967fd74, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T17:00:50,006 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 696d98f18be792cdc1435b3b0e2fd21e in 1265ms, sequenceid=292, compaction requested=false 2024-11-15T17:00:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-15T17:00:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-15T17:00:50,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-15T17:00:50,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-15T17:00:50,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.0430 sec 2024-11-15T17:00:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T17:00:50,116 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:50,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:00:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-15T17:00:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T17:00:50,119 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:00:50,120 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:00:50,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:00:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:50,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:50,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:00:50,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f89743d4f16746b4b999301e64575e25_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690050178/Put/seqid=0 2024-11-15T17:00:50,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742419_1595 (size=17534) 2024-11-15T17:00:50,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690110221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T17:00:50,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690110223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690110335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690110335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T17:00:50,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731690110495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,498 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:50,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:00:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58526 deadline: 1731690110516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,518 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:50,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58506 deadline: 1731690110516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,519 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a9bde5b3a1b5,35119,1731689872090 for 'test_row_1' in 696d98f18be792cdc1435b3b0e2fd21e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:00:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690110545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690110545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:50,597 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f89743d4f16746b4b999301e64575e25_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f89743d4f16746b4b999301e64575e25_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:50,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/618758edb9b145acb5f37a5a0cc4ac23, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:50,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/618758edb9b145acb5f37a5a0cc4ac23 is 175, key is test_row_0/A:col10/1731690050178/Put/seqid=0 2024-11-15T17:00:50,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742420_1596 (size=48639) 2024-11-15T17:00:50,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T17:00:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690110856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690110856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:50,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:50,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:50,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/618758edb9b145acb5f37a5a0cc4ac23 2024-11-15T17:00:51,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/56bcab8d4d6547438680dea694f7a328 is 50, key is test_row_0/B:col10/1731690050178/Put/seqid=0 2024-11-15T17:00:51,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742421_1597 (size=12301) 2024-11-15T17:00:51,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:51,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T17:00:51,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690111366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690111366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:51,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/56bcab8d4d6547438680dea694f7a328 2024-11-15T17:00:51,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/f6f073acebc3430d8e52ba93cb8e246c is 50, key is test_row_0/C:col10/1731690050178/Put/seqid=0 2024-11-15T17:00:51,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742422_1598 (size=12301) 2024-11-15T17:00:51,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:51,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:51,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:51,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:51,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:51,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:00:51,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/f6f073acebc3430d8e52ba93cb8e246c 2024-11-15T17:00:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/618758edb9b145acb5f37a5a0cc4ac23 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/618758edb9b145acb5f37a5a0cc4ac23 2024-11-15T17:00:51,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/618758edb9b145acb5f37a5a0cc4ac23, entries=250, sequenceid=308, filesize=47.5 K 2024-11-15T17:00:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/56bcab8d4d6547438680dea694f7a328 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/56bcab8d4d6547438680dea694f7a328 2024-11-15T17:00:51,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/56bcab8d4d6547438680dea694f7a328, entries=150, sequenceid=308, filesize=12.0 K 2024-11-15T17:00:51,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/f6f073acebc3430d8e52ba93cb8e246c as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f6f073acebc3430d8e52ba93cb8e246c 2024-11-15T17:00:51,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f6f073acebc3430d8e52ba93cb8e246c, entries=150, sequenceid=308, filesize=12.0 K 2024-11-15T17:00:51,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 696d98f18be792cdc1435b3b0e2fd21e in 1664ms, sequenceid=308, compaction requested=true 2024-11-15T17:00:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:51,842 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:51,842 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:51,843 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:51,843 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:51,843 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,843 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111854 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:51,843 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/78308d5672a5486590c842ba7880317d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/412f32efe57a40e6ad85a6e83c84bc1b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/56bcab8d4d6547438680dea694f7a328] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=36.6 K 2024-11-15T17:00:51,843 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:51,843 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,843 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f297678a0b6f4fb7add93701fb7b5dbe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/168b5a0d8be64f8b98f2b10d02703957, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/618758edb9b145acb5f37a5a0cc4ac23] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=109.2 K 2024-11-15T17:00:51,843 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,844 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f297678a0b6f4fb7add93701fb7b5dbe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/168b5a0d8be64f8b98f2b10d02703957, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/618758edb9b145acb5f37a5a0cc4ac23] 2024-11-15T17:00:51,844 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 78308d5672a5486590c842ba7880317d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731690045630 2024-11-15T17:00:51,844 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 412f32efe57a40e6ad85a6e83c84bc1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731690047866 2024-11-15T17:00:51,844 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting f297678a0b6f4fb7add93701fb7b5dbe, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731690045630 2024-11-15T17:00:51,844 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 168b5a0d8be64f8b98f2b10d02703957, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731690047866 2024-11-15T17:00:51,844 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 56bcab8d4d6547438680dea694f7a328, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731690049033 2024-11-15T17:00:51,844 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 618758edb9b145acb5f37a5a0cc4ac23, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731690049029 2024-11-15T17:00:51,851 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:51,851 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/8b0a6e770bab429a9544d0921d8f9f93 is 50, key is test_row_0/B:col10/1731690050178/Put/seqid=0 2024-11-15T17:00:51,852 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:51,854 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111557f5098322264df79cc4f72747403e36_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:51,856 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111557f5098322264df79cc4f72747403e36_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:51,856 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111557f5098322264df79cc4f72747403e36_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:51,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742423_1599 (size=13051) 2024-11-15T17:00:51,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742424_1600 (size=4469) 2024-11-15T17:00:51,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35119 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T17:00:51,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:51,964 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:00:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a6580becaef94750be152516617041d0_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690050219/Put/seqid=0 2024-11-15T17:00:51,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742425_1601 (size=12454) 2024-11-15T17:00:52,261 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/8b0a6e770bab429a9544d0921d8f9f93 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/8b0a6e770bab429a9544d0921d8f9f93 2024-11-15T17:00:52,265 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into 8b0a6e770bab429a9544d0921d8f9f93(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:52,265 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:52,265 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=13, startTime=1731690051842; duration=0sec 2024-11-15T17:00:52,265 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T17:00:52,265 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:52,265 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:52,266 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:52,266 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:52,266 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:52,266 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/0daa56d1c8e644869d85bc9c6819e798, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a95e6fe9cc1f4aa287ca072ae967fd74, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f6f073acebc3430d8e52ba93cb8e246c] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=36.6 K 2024-11-15T17:00:52,267 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 0daa56d1c8e644869d85bc9c6819e798, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731690045630 2024-11-15T17:00:52,267 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting a95e6fe9cc1f4aa287ca072ae967fd74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731690047866 2024-11-15T17:00:52,267 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f073acebc3430d8e52ba93cb8e246c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731690049033 2024-11-15T17:00:52,273 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#502 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:52,273 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:52,273 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/4a367439f7734f24b28816cea7ad623e is 175, key is test_row_0/A:col10/1731690050178/Put/seqid=0 2024-11-15T17:00:52,273 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/aa6d829dc42e4d2b87714b9e4ef1bd90 is 50, key is test_row_0/C:col10/1731690050178/Put/seqid=0 2024-11-15T17:00:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742426_1602 (size=32005) 2024-11-15T17:00:52,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742427_1603 (size=13051) 2024-11-15T17:00:52,294 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/4a367439f7734f24b28816cea7ad623e as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/4a367439f7734f24b28816cea7ad623e 2024-11-15T17:00:52,297 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into 4a367439f7734f24b28816cea7ad623e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:52,298 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:52,298 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=13, startTime=1731690051842; duration=0sec 2024-11-15T17:00:52,298 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:52,298 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:52,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:52,377 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a6580becaef94750be152516617041d0_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a6580becaef94750be152516617041d0_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. as already flushing 2024-11-15T17:00:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:52,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/a6a0d5810d5949f697b0d58a5e4796ab, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:52,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/a6a0d5810d5949f697b0d58a5e4796ab is 175, key is test_row_0/A:col10/1731690050219/Put/seqid=0 2024-11-15T17:00:52,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742428_1604 (size=31255) 2024-11-15T17:00:52,383 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/a6a0d5810d5949f697b0d58a5e4796ab 2024-11-15T17:00:52,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/75278998b76840c49aaff24d466ab1e2 is 50, key is test_row_0/B:col10/1731690050219/Put/seqid=0 2024-11-15T17:00:52,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742429_1605 (size=12301) 2024-11-15T17:00:52,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690112407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:52,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690112409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690112515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690112515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:52,695 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/aa6d829dc42e4d2b87714b9e4ef1bd90 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/aa6d829dc42e4d2b87714b9e4ef1bd90 2024-11-15T17:00:52,700 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into aa6d829dc42e4d2b87714b9e4ef1bd90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:52,700 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:52,700 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=13, startTime=1731690051842; duration=0sec 2024-11-15T17:00:52,700 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:52,700 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:00:52,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58486 deadline: 1731690112725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:52,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:00:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58496 deadline: 1731690112725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:00:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 , the old value is region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:00:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., hostname=a9bde5b3a1b5,35119,1731689872090, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:00:52,804 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/75278998b76840c49aaff24d466ab1e2 2024-11-15T17:00:52,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/434c696afb0a474ea85d8b7073d98955 is 50, key is test_row_0/C:col10/1731690050219/Put/seqid=0 2024-11-15T17:00:52,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742430_1606 (size=12301) 2024-11-15T17:00:52,815 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/434c696afb0a474ea85d8b7073d98955 2024-11-15T17:00:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/a6a0d5810d5949f697b0d58a5e4796ab as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/a6a0d5810d5949f697b0d58a5e4796ab 2024-11-15T17:00:52,822 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/a6a0d5810d5949f697b0d58a5e4796ab, entries=150, sequenceid=331, filesize=30.5 K 2024-11-15T17:00:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/75278998b76840c49aaff24d466ab1e2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/75278998b76840c49aaff24d466ab1e2 2024-11-15T17:00:52,826 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/75278998b76840c49aaff24d466ab1e2, entries=150, sequenceid=331, filesize=12.0 K 2024-11-15T17:00:52,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/434c696afb0a474ea85d8b7073d98955 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/434c696afb0a474ea85d8b7073d98955 2024-11-15T17:00:52,830 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/434c696afb0a474ea85d8b7073d98955, entries=150, sequenceid=331, filesize=12.0 K 2024-11-15T17:00:52,831 INFO [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 696d98f18be792cdc1435b3b0e2fd21e in 867ms, sequenceid=331, compaction requested=false 2024-11-15T17:00:52,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:52,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:52,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a9bde5b3a1b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-15T17:00:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-15T17:00:52,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-15T17:00:52,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7120 sec 2024-11-15T17:00:52,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.7180 sec 2024-11-15T17:00:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35119 {}] regionserver.HRegion(8855): Flush requested on 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:53,038 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:00:53,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:00:53,039 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-15T17:00:53,039 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:53,039 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,039 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:53,039 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-11-15T17:00:53,039 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:53,039 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,039 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:53,041 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-15T17:00:53,041 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:53,041 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,041 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:53,043 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-11-15T17:00:53,043 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:53,043 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,043 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:53,043 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-11-15T17:00:53,043 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:53,043 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,043 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,043 INFO [Thread-2300 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2300. 2024-11-15T17:00:53,043 DEBUG [Thread-2300 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:53,043 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:53,043 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:53,044 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-15T17:00:53,044 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:00:53,044 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,044 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:00:53,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:00:53,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111508c6723cfd1e4d4991214b2520c3d3aa_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690052390/Put/seqid=0 2024-11-15T17:00:53,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742431_1607 (size=17534) 2024-11-15T17:00:53,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:00:53,456 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111508c6723cfd1e4d4991214b2520c3d3aa_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111508c6723cfd1e4d4991214b2520c3d3aa_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:00:53,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/93158c8fa1ca4d4ea0ae558a45e2e7d3, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:53,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/93158c8fa1ca4d4ea0ae558a45e2e7d3 is 175, key is test_row_0/A:col10/1731690052390/Put/seqid=0 2024-11-15T17:00:53,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742432_1608 (size=48639) 2024-11-15T17:00:53,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/93158c8fa1ca4d4ea0ae558a45e2e7d3 2024-11-15T17:00:53,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/28d62a8ae9a644289a74ad1b8cf504e0 is 50, key is test_row_0/B:col10/1731690052390/Put/seqid=0 2024-11-15T17:00:53,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742433_1609 (size=12301) 2024-11-15T17:00:54,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/28d62a8ae9a644289a74ad1b8cf504e0 2024-11-15T17:00:54,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/5d4791c296cc46c09d5048ef63b6d7bb is 50, key is test_row_0/C:col10/1731690052390/Put/seqid=0 2024-11-15T17:00:54,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742434_1610 (size=12301) 2024-11-15T17:00:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T17:00:54,296 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:00:54,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/5d4791c296cc46c09d5048ef63b6d7bb 2024-11-15T17:00:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/93158c8fa1ca4d4ea0ae558a45e2e7d3 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/93158c8fa1ca4d4ea0ae558a45e2e7d3 2024-11-15T17:00:54,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/93158c8fa1ca4d4ea0ae558a45e2e7d3, entries=250, sequenceid=349, filesize=47.5 K 2024-11-15T17:00:54,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/28d62a8ae9a644289a74ad1b8cf504e0 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/28d62a8ae9a644289a74ad1b8cf504e0 2024-11-15T17:00:54,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/28d62a8ae9a644289a74ad1b8cf504e0, entries=150, sequenceid=349, filesize=12.0 K 2024-11-15T17:00:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/5d4791c296cc46c09d5048ef63b6d7bb as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d4791c296cc46c09d5048ef63b6d7bb 2024-11-15T17:00:54,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d4791c296cc46c09d5048ef63b6d7bb, entries=150, sequenceid=349, filesize=12.0 K 2024-11-15T17:00:54,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=6.71 KB/6870 for 696d98f18be792cdc1435b3b0e2fd21e in 1654ms, sequenceid=349, compaction requested=true 2024-11-15T17:00:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:00:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:00:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:54,692 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 696d98f18be792cdc1435b3b0e2fd21e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:00:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:54,692 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/A is initiating minor compaction (all files) 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:54,693 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/A in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/B is initiating minor compaction (all files) 2024-11-15T17:00:54,693 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/4a367439f7734f24b28816cea7ad623e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/a6a0d5810d5949f697b0d58a5e4796ab, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/93158c8fa1ca4d4ea0ae558a45e2e7d3] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=109.3 K 2024-11-15T17:00:54,693 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/B in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:54,693 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. files: [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/4a367439f7734f24b28816cea7ad623e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/a6a0d5810d5949f697b0d58a5e4796ab, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/93158c8fa1ca4d4ea0ae558a45e2e7d3] 2024-11-15T17:00:54,693 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/8b0a6e770bab429a9544d0921d8f9f93, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/75278998b76840c49aaff24d466ab1e2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/28d62a8ae9a644289a74ad1b8cf504e0] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=36.8 K 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b0a6e770bab429a9544d0921d8f9f93, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731690049033 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a367439f7734f24b28816cea7ad623e, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731690049033 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 75278998b76840c49aaff24d466ab1e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731690050210 2024-11-15T17:00:54,693 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6a0d5810d5949f697b0d58a5e4796ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731690050210 2024-11-15T17:00:54,694 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d62a8ae9a644289a74ad1b8cf504e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731690052390 2024-11-15T17:00:54,694 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93158c8fa1ca4d4ea0ae558a45e2e7d3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731690052390 2024-11-15T17:00:54,699 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:54,701 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#B#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:54,701 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/a63b15efc32143c1be16f95bb97ee999 is 50, key is test_row_0/B:col10/1731690052390/Put/seqid=0 2024-11-15T17:00:54,702 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115543684cba0324d2ab24a2faa2e8d63a4_696d98f18be792cdc1435b3b0e2fd21e store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:54,705 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115543684cba0324d2ab24a2faa2e8d63a4_696d98f18be792cdc1435b3b0e2fd21e, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:54,705 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115543684cba0324d2ab24a2faa2e8d63a4_696d98f18be792cdc1435b3b0e2fd21e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:00:54,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742435_1611 (size=13153) 2024-11-15T17:00:54,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742436_1612 (size=4469) 2024-11-15T17:00:55,110 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:55,110 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2b9d8c7c74074577b8c8bb2aa921d911 is 175, key is test_row_0/A:col10/1731690052390/Put/seqid=0 2024-11-15T17:00:55,110 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/a63b15efc32143c1be16f95bb97ee999 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a63b15efc32143c1be16f95bb97ee999 2024-11-15T17:00:55,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742437_1613 (size=32107) 2024-11-15T17:00:55,114 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/B of 696d98f18be792cdc1435b3b0e2fd21e into a63b15efc32143c1be16f95bb97ee999(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:55,114 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:55,114 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/B, priority=13, startTime=1731690054692; duration=0sec 2024-11-15T17:00:55,114 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:00:55,114 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:B 2024-11-15T17:00:55,114 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:00:55,115 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:00:55,115 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1541): 696d98f18be792cdc1435b3b0e2fd21e/C is initiating minor compaction (all files) 2024-11-15T17:00:55,115 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 696d98f18be792cdc1435b3b0e2fd21e/C in TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:00:55,115 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/aa6d829dc42e4d2b87714b9e4ef1bd90, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/434c696afb0a474ea85d8b7073d98955, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d4791c296cc46c09d5048ef63b6d7bb] into tmpdir=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp, totalSize=36.8 K 2024-11-15T17:00:55,115 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting aa6d829dc42e4d2b87714b9e4ef1bd90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731690049033 2024-11-15T17:00:55,116 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 434c696afb0a474ea85d8b7073d98955, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731690050210 2024-11-15T17:00:55,116 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4791c296cc46c09d5048ef63b6d7bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731690052390 2024-11-15T17:00:55,121 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 696d98f18be792cdc1435b3b0e2fd21e#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:00:55,121 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/fe374ed3054c4f358d266355be5191fa is 50, key is test_row_0/C:col10/1731690052390/Put/seqid=0 2024-11-15T17:00:55,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742438_1614 (size=13153) 2024-11-15T17:00:55,517 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2b9d8c7c74074577b8c8bb2aa921d911 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2b9d8c7c74074577b8c8bb2aa921d911 2024-11-15T17:00:55,521 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/A of 696d98f18be792cdc1435b3b0e2fd21e into 2b9d8c7c74074577b8c8bb2aa921d911(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:55,521 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:55,521 INFO [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/A, priority=13, startTime=1731690054692; duration=0sec 2024-11-15T17:00:55,521 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:55,521 DEBUG [RS:0;a9bde5b3a1b5:35119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:A 2024-11-15T17:00:55,527 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/fe374ed3054c4f358d266355be5191fa as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe374ed3054c4f358d266355be5191fa 2024-11-15T17:00:55,531 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 696d98f18be792cdc1435b3b0e2fd21e/C of 696d98f18be792cdc1435b3b0e2fd21e into fe374ed3054c4f358d266355be5191fa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:00:55,531 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 696d98f18be792cdc1435b3b0e2fd21e: 2024-11-15T17:00:55,531 INFO [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., storeName=696d98f18be792cdc1435b3b0e2fd21e/C, priority=13, startTime=1731690054692; duration=0sec 2024-11-15T17:00:55,531 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:00:55,531 DEBUG [RS:0;a9bde5b3a1b5:35119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 696d98f18be792cdc1435b3b0e2fd21e:C 2024-11-15T17:01:00,558 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-11-15T17:01:00,558 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:01:00,558 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,558 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-11-15T17:01:00,558 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,558 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:01:00,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:01:00,558 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,558 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:01:00,588 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-11-15T17:01:00,588 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:01:00,588 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,588 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,588 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3710 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3521 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3528 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3719 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3523 2024-11-15T17:01:00,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:01:00,588 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:01:00,588 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:01:00,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:00,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:01:00,589 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:01:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:01:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T17:01:00,592 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731690060592"}]},"ts":"1731690060592"} 2024-11-15T17:01:00,593 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:01:00,593 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:01:00,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:01:00,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, UNASSIGN}] 2024-11-15T17:01:00,595 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, UNASSIGN 2024-11-15T17:01:00,596 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=CLOSING, regionLocation=a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:01:00,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, UNASSIGN because future has completed 2024-11-15T17:01:00,597 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:01:00,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090}] 2024-11-15T17:01:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T17:01:00,749 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(122): Close 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:00,749 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:01:00,750 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1722): Closing 696d98f18be792cdc1435b3b0e2fd21e, disabling compactions & flushes 2024-11-15T17:01:00,750 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:01:00,750 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:01:00,750 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. after waiting 0 ms 2024-11-15T17:01:00,750 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:01:00,750 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(2902): Flushing 696d98f18be792cdc1435b3b0e2fd21e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T17:01:00,754 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151196b049830c457ca1c840e8ef138283_696d98f18be792cdc1435b3b0e2fd21e is 50, key is test_row_0/A:col10/1731690053041/Put/seqid=0 2024-11-15T17:01:00,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742439_1615 (size=9914) 2024-11-15T17:01:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T17:01:01,158 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:01:01,162 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151196b049830c457ca1c840e8ef138283_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151196b049830c457ca1c840e8ef138283_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:01,162 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ce76bb7e61b4d159b65d18061afd15b, store: [table=TestAcidGuarantees family=A region=696d98f18be792cdc1435b3b0e2fd21e] 2024-11-15T17:01:01,163 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ce76bb7e61b4d159b65d18061afd15b is 175, key is test_row_0/A:col10/1731690053041/Put/seqid=0 2024-11-15T17:01:01,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742440_1616 (size=22561) 2024-11-15T17:01:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T17:01:01,566 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ce76bb7e61b4d159b65d18061afd15b 2024-11-15T17:01:01,572 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/be80309ed2f042da811c0360cbb338e2 is 50, key is test_row_0/B:col10/1731690053041/Put/seqid=0 2024-11-15T17:01:01,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742441_1617 (size=9857) 2024-11-15T17:01:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T17:01:01,975 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/be80309ed2f042da811c0360cbb338e2 2024-11-15T17:01:01,980 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8bcd2dcd2c934d83ab1bf2cc0b738d93 is 50, key is test_row_0/C:col10/1731690053041/Put/seqid=0 2024-11-15T17:01:01,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742442_1618 (size=9857) 2024-11-15T17:01:02,384 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8bcd2dcd2c934d83ab1bf2cc0b738d93 2024-11-15T17:01:02,387 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/A/2ce76bb7e61b4d159b65d18061afd15b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ce76bb7e61b4d159b65d18061afd15b 2024-11-15T17:01:02,390 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ce76bb7e61b4d159b65d18061afd15b, entries=100, sequenceid=359, filesize=22.0 K 2024-11-15T17:01:02,391 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/B/be80309ed2f042da811c0360cbb338e2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/be80309ed2f042da811c0360cbb338e2 2024-11-15T17:01:02,393 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/be80309ed2f042da811c0360cbb338e2, entries=100, sequenceid=359, filesize=9.6 K 2024-11-15T17:01:02,394 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/.tmp/C/8bcd2dcd2c934d83ab1bf2cc0b738d93 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8bcd2dcd2c934d83ab1bf2cc0b738d93 2024-11-15T17:01:02,396 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8bcd2dcd2c934d83ab1bf2cc0b738d93, entries=100, sequenceid=359, filesize=9.6 K 2024-11-15T17:01:02,397 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 696d98f18be792cdc1435b3b0e2fd21e in 1647ms, sequenceid=359, compaction requested=false 2024-11-15T17:01:02,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2d13a2d653434761a0735494dfa52bfd, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f1b412573e2549e78e08d43b6b021a4d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/c9235298d8554643b8915f97b77a77dc, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/59e81cc4516c49f097ed5cfb2be76516, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/27a8d8f06f874d77a032fa66f7603147, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/1438cc9bd03f47d0a70f03a24392b328, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/6d2afbd03180476ebfbd70c6ce06ea7c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/faa2d7ee2a314d71b8c9cc5eb7df0563, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/659cec8db1e6422fa5bb31cb9d24364d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/b3109914f1004441914299a74e7247ac, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/752b74df24fb454d8c133fce34386167, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/447a7d5625ee4ce29250daf9db666ad9, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/66bf8b96ce0a43db9146463015434ea3, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ab94d18ebab46299822b99550dafd90, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ce9581c1504a44b293bda3bf88a26ca8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/5735dfb3eb3a4df4959d076325ad13ec, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/0b348c6a9c1840d4b84749abaa59f315, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/fe0021ee9b484bf4aa0305448bad6582, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ef13dc2d85ab4109b342af0bb32e0962, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f297678a0b6f4fb7add93701fb7b5dbe, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/168b5a0d8be64f8b98f2b10d02703957, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/618758edb9b145acb5f37a5a0cc4ac23, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/4a367439f7734f24b28816cea7ad623e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/a6a0d5810d5949f697b0d58a5e4796ab, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/93158c8fa1ca4d4ea0ae558a45e2e7d3] to archive 2024-11-15T17:01:02,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:01:02,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2d13a2d653434761a0735494dfa52bfd to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2d13a2d653434761a0735494dfa52bfd 2024-11-15T17:01:02,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f1b412573e2549e78e08d43b6b021a4d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f1b412573e2549e78e08d43b6b021a4d 2024-11-15T17:01:02,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/c9235298d8554643b8915f97b77a77dc to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/c9235298d8554643b8915f97b77a77dc 2024-11-15T17:01:02,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/59e81cc4516c49f097ed5cfb2be76516 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/59e81cc4516c49f097ed5cfb2be76516 2024-11-15T17:01:02,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/27a8d8f06f874d77a032fa66f7603147 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/27a8d8f06f874d77a032fa66f7603147 2024-11-15T17:01:02,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/1438cc9bd03f47d0a70f03a24392b328 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/1438cc9bd03f47d0a70f03a24392b328 2024-11-15T17:01:02,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/6d2afbd03180476ebfbd70c6ce06ea7c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/6d2afbd03180476ebfbd70c6ce06ea7c 2024-11-15T17:01:02,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/faa2d7ee2a314d71b8c9cc5eb7df0563 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/faa2d7ee2a314d71b8c9cc5eb7df0563 2024-11-15T17:01:02,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/659cec8db1e6422fa5bb31cb9d24364d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/659cec8db1e6422fa5bb31cb9d24364d 2024-11-15T17:01:02,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/b3109914f1004441914299a74e7247ac to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/b3109914f1004441914299a74e7247ac 2024-11-15T17:01:02,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/752b74df24fb454d8c133fce34386167 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/752b74df24fb454d8c133fce34386167 2024-11-15T17:01:02,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/447a7d5625ee4ce29250daf9db666ad9 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/447a7d5625ee4ce29250daf9db666ad9 2024-11-15T17:01:02,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/66bf8b96ce0a43db9146463015434ea3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/66bf8b96ce0a43db9146463015434ea3 2024-11-15T17:01:02,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ab94d18ebab46299822b99550dafd90 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ab94d18ebab46299822b99550dafd90 2024-11-15T17:01:02,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ce9581c1504a44b293bda3bf88a26ca8 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ce9581c1504a44b293bda3bf88a26ca8 2024-11-15T17:01:02,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/5735dfb3eb3a4df4959d076325ad13ec to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/5735dfb3eb3a4df4959d076325ad13ec 2024-11-15T17:01:02,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/0b348c6a9c1840d4b84749abaa59f315 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/0b348c6a9c1840d4b84749abaa59f315 2024-11-15T17:01:02,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/fe0021ee9b484bf4aa0305448bad6582 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/fe0021ee9b484bf4aa0305448bad6582 2024-11-15T17:01:02,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ef13dc2d85ab4109b342af0bb32e0962 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/ef13dc2d85ab4109b342af0bb32e0962 2024-11-15T17:01:02,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f297678a0b6f4fb7add93701fb7b5dbe to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/f297678a0b6f4fb7add93701fb7b5dbe 2024-11-15T17:01:02,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/168b5a0d8be64f8b98f2b10d02703957 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/168b5a0d8be64f8b98f2b10d02703957 2024-11-15T17:01:02,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/618758edb9b145acb5f37a5a0cc4ac23 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/618758edb9b145acb5f37a5a0cc4ac23 2024-11-15T17:01:02,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/4a367439f7734f24b28816cea7ad623e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/4a367439f7734f24b28816cea7ad623e 2024-11-15T17:01:02,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/a6a0d5810d5949f697b0d58a5e4796ab to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/a6a0d5810d5949f697b0d58a5e4796ab 2024-11-15T17:01:02,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/93158c8fa1ca4d4ea0ae558a45e2e7d3 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/93158c8fa1ca4d4ea0ae558a45e2e7d3 2024-11-15T17:01:02,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1818bafadbd54ef6a25a85c2f7ade75c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/43747ef607834f7790a6d0ac3aa71fe8, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/d6cf0dac506743faa707fdaac182966a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c3211f5378e34fc7b87f7e554eda3eb5, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1fb9c47a9f1942deb7bc3183ea73e092, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a3b529d703c3462d91f2cc1718fabf74, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/9107154f7a87480db09bc129d0dea7ee, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c2e1d3469f4f48f79a0eeeaa352b809b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/4a86b351556b4be2a7ddb1def1d806db, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/cab12cbc42944ced9f59bb724bea57a7, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ca1b59b165a7408d80df691aeb255c31, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ab8e2cc07ada4cdd969f998e3772d0f2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/04f97c1a6b604d118df2be0f1bcaba4a, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/076b976ce1f94e818d14cea497060928, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a33cf4e15f084064851df0d7af05b886, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/30bb1ed100224a61ad0fdf867e7a953b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/bbe9db1ae16b4723976990f8d30c0952, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/3c9f01c2b8b84b21a1fa93818aaff168, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/78308d5672a5486590c842ba7880317d, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ae7fedadd4ff4dafba00919a50ab8799, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/412f32efe57a40e6ad85a6e83c84bc1b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/8b0a6e770bab429a9544d0921d8f9f93, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/56bcab8d4d6547438680dea694f7a328, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/75278998b76840c49aaff24d466ab1e2, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/28d62a8ae9a644289a74ad1b8cf504e0] to archive 2024-11-15T17:01:02,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:01:02,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1818bafadbd54ef6a25a85c2f7ade75c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1818bafadbd54ef6a25a85c2f7ade75c 2024-11-15T17:01:02,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/43747ef607834f7790a6d0ac3aa71fe8 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/43747ef607834f7790a6d0ac3aa71fe8 2024-11-15T17:01:02,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/d6cf0dac506743faa707fdaac182966a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/d6cf0dac506743faa707fdaac182966a 2024-11-15T17:01:02,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c3211f5378e34fc7b87f7e554eda3eb5 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c3211f5378e34fc7b87f7e554eda3eb5 2024-11-15T17:01:02,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1fb9c47a9f1942deb7bc3183ea73e092 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/1fb9c47a9f1942deb7bc3183ea73e092 2024-11-15T17:01:02,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a3b529d703c3462d91f2cc1718fabf74 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a3b529d703c3462d91f2cc1718fabf74 2024-11-15T17:01:02,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/9107154f7a87480db09bc129d0dea7ee to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/9107154f7a87480db09bc129d0dea7ee 2024-11-15T17:01:02,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c2e1d3469f4f48f79a0eeeaa352b809b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/c2e1d3469f4f48f79a0eeeaa352b809b 2024-11-15T17:01:02,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/4a86b351556b4be2a7ddb1def1d806db to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/4a86b351556b4be2a7ddb1def1d806db 2024-11-15T17:01:02,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/cab12cbc42944ced9f59bb724bea57a7 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/cab12cbc42944ced9f59bb724bea57a7 2024-11-15T17:01:02,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ca1b59b165a7408d80df691aeb255c31 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ca1b59b165a7408d80df691aeb255c31 2024-11-15T17:01:02,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ab8e2cc07ada4cdd969f998e3772d0f2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ab8e2cc07ada4cdd969f998e3772d0f2 2024-11-15T17:01:02,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/04f97c1a6b604d118df2be0f1bcaba4a to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/04f97c1a6b604d118df2be0f1bcaba4a 2024-11-15T17:01:02,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/076b976ce1f94e818d14cea497060928 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/076b976ce1f94e818d14cea497060928 2024-11-15T17:01:02,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a33cf4e15f084064851df0d7af05b886 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a33cf4e15f084064851df0d7af05b886 2024-11-15T17:01:02,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/30bb1ed100224a61ad0fdf867e7a953b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/30bb1ed100224a61ad0fdf867e7a953b 2024-11-15T17:01:02,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/bbe9db1ae16b4723976990f8d30c0952 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/bbe9db1ae16b4723976990f8d30c0952 2024-11-15T17:01:02,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/3c9f01c2b8b84b21a1fa93818aaff168 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/3c9f01c2b8b84b21a1fa93818aaff168 2024-11-15T17:01:02,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/78308d5672a5486590c842ba7880317d to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/78308d5672a5486590c842ba7880317d 2024-11-15T17:01:02,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ae7fedadd4ff4dafba00919a50ab8799 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/ae7fedadd4ff4dafba00919a50ab8799 2024-11-15T17:01:02,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/412f32efe57a40e6ad85a6e83c84bc1b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/412f32efe57a40e6ad85a6e83c84bc1b 2024-11-15T17:01:02,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/8b0a6e770bab429a9544d0921d8f9f93 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/8b0a6e770bab429a9544d0921d8f9f93 2024-11-15T17:01:02,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/56bcab8d4d6547438680dea694f7a328 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/56bcab8d4d6547438680dea694f7a328 2024-11-15T17:01:02,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/75278998b76840c49aaff24d466ab1e2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/75278998b76840c49aaff24d466ab1e2 2024-11-15T17:01:02,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/28d62a8ae9a644289a74ad1b8cf504e0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/28d62a8ae9a644289a74ad1b8cf504e0 2024-11-15T17:01:02,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/4e2dad1e87e141c3b8b7da92db7bd642, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e6fc65ce550a47a28ef03431235933c1, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8c008bea96df4709b1f0d355dea85f6e, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/84f87c62e8a948408707585b3086e61b, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/019e1647642c423690d77db16b248372, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a55e6c9593254941bca99d7a5cb03b2c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2c8828de633a428abe2fe79b7741efb0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d069544bd8046ef9b7e077fd9840523, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/833ce57303b240c9bbf7d3fd56cbaac4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/7f940e943bc94408aaf5747663fbe6e4, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/15e2856b0a494e98936a827c7e1b8931, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e02583a951894f0388cda208fcc785fa, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/53065d2d5262461e841c01943c0363f0, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a367f397e29b40e7a7d9d1eb538c8d66, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f9b60b0211d1454288f0e407a9e5d255, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/6d13818e3b89420cb7cfba8e1b4fe00c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe64abed29e54f9db1fa96e250cea760, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8fbd9bd85a1048e3a6f6aa8b1c43f017, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/0daa56d1c8e644869d85bc9c6819e798, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2906c374dd934bd7a0d5c3375a8cab06, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a95e6fe9cc1f4aa287ca072ae967fd74, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/aa6d829dc42e4d2b87714b9e4ef1bd90, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f6f073acebc3430d8e52ba93cb8e246c, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/434c696afb0a474ea85d8b7073d98955, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d4791c296cc46c09d5048ef63b6d7bb] to archive 2024-11-15T17:01:02,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:01:02,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/4e2dad1e87e141c3b8b7da92db7bd642 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/4e2dad1e87e141c3b8b7da92db7bd642 2024-11-15T17:01:02,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e6fc65ce550a47a28ef03431235933c1 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e6fc65ce550a47a28ef03431235933c1 2024-11-15T17:01:02,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8c008bea96df4709b1f0d355dea85f6e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8c008bea96df4709b1f0d355dea85f6e 2024-11-15T17:01:02,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/84f87c62e8a948408707585b3086e61b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/84f87c62e8a948408707585b3086e61b 2024-11-15T17:01:02,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/019e1647642c423690d77db16b248372 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/019e1647642c423690d77db16b248372 2024-11-15T17:01:02,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a55e6c9593254941bca99d7a5cb03b2c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a55e6c9593254941bca99d7a5cb03b2c 2024-11-15T17:01:02,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2c8828de633a428abe2fe79b7741efb0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2c8828de633a428abe2fe79b7741efb0 2024-11-15T17:01:02,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d069544bd8046ef9b7e077fd9840523 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d069544bd8046ef9b7e077fd9840523 2024-11-15T17:01:02,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/833ce57303b240c9bbf7d3fd56cbaac4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/833ce57303b240c9bbf7d3fd56cbaac4 2024-11-15T17:01:02,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/7f940e943bc94408aaf5747663fbe6e4 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/7f940e943bc94408aaf5747663fbe6e4 2024-11-15T17:01:02,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/15e2856b0a494e98936a827c7e1b8931 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/15e2856b0a494e98936a827c7e1b8931 2024-11-15T17:01:02,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e02583a951894f0388cda208fcc785fa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/e02583a951894f0388cda208fcc785fa 2024-11-15T17:01:02,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/53065d2d5262461e841c01943c0363f0 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/53065d2d5262461e841c01943c0363f0 2024-11-15T17:01:02,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a367f397e29b40e7a7d9d1eb538c8d66 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a367f397e29b40e7a7d9d1eb538c8d66 2024-11-15T17:01:02,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f9b60b0211d1454288f0e407a9e5d255 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f9b60b0211d1454288f0e407a9e5d255 2024-11-15T17:01:02,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/6d13818e3b89420cb7cfba8e1b4fe00c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/6d13818e3b89420cb7cfba8e1b4fe00c 2024-11-15T17:01:02,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe64abed29e54f9db1fa96e250cea760 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe64abed29e54f9db1fa96e250cea760 2024-11-15T17:01:02,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8fbd9bd85a1048e3a6f6aa8b1c43f017 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8fbd9bd85a1048e3a6f6aa8b1c43f017 2024-11-15T17:01:02,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/0daa56d1c8e644869d85bc9c6819e798 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/0daa56d1c8e644869d85bc9c6819e798 2024-11-15T17:01:02,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2906c374dd934bd7a0d5c3375a8cab06 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/2906c374dd934bd7a0d5c3375a8cab06 2024-11-15T17:01:02,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a95e6fe9cc1f4aa287ca072ae967fd74 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/a95e6fe9cc1f4aa287ca072ae967fd74 2024-11-15T17:01:02,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/aa6d829dc42e4d2b87714b9e4ef1bd90 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/aa6d829dc42e4d2b87714b9e4ef1bd90 2024-11-15T17:01:02,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f6f073acebc3430d8e52ba93cb8e246c to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/f6f073acebc3430d8e52ba93cb8e246c 2024-11-15T17:01:02,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/434c696afb0a474ea85d8b7073d98955 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/434c696afb0a474ea85d8b7073d98955 2024-11-15T17:01:02,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d4791c296cc46c09d5048ef63b6d7bb to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/5d4791c296cc46c09d5048ef63b6d7bb 2024-11-15T17:01:02,462 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=4 2024-11-15T17:01:02,463 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. 2024-11-15T17:01:02,463 DEBUG [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1676): Region close journal for 696d98f18be792cdc1435b3b0e2fd21e: Waiting for close lock at 1731690060750Running coprocessor pre-close hooks at 1731690060750Disabling compacts and flushes for region at 1731690060750Disabling writes for close at 1731690060750Obtaining lock to block concurrent updates at 1731690060750Preparing flush snapshotting stores in 696d98f18be792cdc1435b3b0e2fd21e at 1731690060750Finished memstore snapshotting TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731690060750Flushing stores of TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e. at 1731690060750Flushing 696d98f18be792cdc1435b3b0e2fd21e/A: creating writer at 1731690060750Flushing mob file 696d98f18be792cdc1435b3b0e2fd21e/A: appending metadata at 1731690060754 (+4 ms)Flushing mob file 696d98f18be792cdc1435b3b0e2fd21e/A: closing flushed file at 1731690060754Flushing 696d98f18be792cdc1435b3b0e2fd21e/A: appending metadata at 1731690061162 (+408 ms)Flushing 696d98f18be792cdc1435b3b0e2fd21e/A: closing flushed file at 1731690061162Flushing 696d98f18be792cdc1435b3b0e2fd21e/B: creating writer at 1731690061569 (+407 ms)Flushing 696d98f18be792cdc1435b3b0e2fd21e/B: appending metadata at 1731690061571 (+2 ms)Flushing 696d98f18be792cdc1435b3b0e2fd21e/B: closing flushed file at 1731690061571Flushing 696d98f18be792cdc1435b3b0e2fd21e/C: creating writer at 1731690061978 (+407 ms)Flushing 696d98f18be792cdc1435b3b0e2fd21e/C: appending metadata at 1731690061980 (+2 ms)Flushing 696d98f18be792cdc1435b3b0e2fd21e/C: closing flushed file at 1731690061980Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bac5840: reopening flushed file at 1731690062387 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b42b7ba: reopening flushed file at 1731690062390 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47da1bc7: reopening flushed file at 1731690062393 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 696d98f18be792cdc1435b3b0e2fd21e in 1647ms, sequenceid=359, compaction requested=false at 1731690062397 (+4 ms)Writing region close event to WAL at 1731690062460 (+63 ms)Running coprocessor post-close hooks at 1731690062462 (+2 ms)Closed at 1731690062463 (+1 ms) 2024-11-15T17:01:02,465 INFO [RS_CLOSE_REGION-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(157): Closed 696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=696d98f18be792cdc1435b3b0e2fd21e, regionState=CLOSED 2024-11-15T17:01:02,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 because future has completed 2024-11-15T17:01:02,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-15T17:01:02,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseRegionProcedure 696d98f18be792cdc1435b3b0e2fd21e, server=a9bde5b3a1b5,35119,1731689872090 in 1.8700 sec 2024-11-15T17:01:02,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=176, resume processing ppid=175 2024-11-15T17:01:02,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, ppid=175, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=696d98f18be792cdc1435b3b0e2fd21e, UNASSIGN in 1.8740 sec 2024-11-15T17:01:02,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-15T17:01:02,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-15T17:01:02,473 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731690062473"}]},"ts":"1731690062473"} 2024-11-15T17:01:02,474 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:01:02,474 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:01:02,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8850 sec 2024-11-15T17:01:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T17:01:02,736 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:01:02,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:01:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:01:02,738 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:01:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-15T17:01:02,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=178, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:01:02,741 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,743 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C, FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/recovered.edits] 2024-11-15T17:01:02,745 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2b9d8c7c74074577b8c8bb2aa921d911 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2b9d8c7c74074577b8c8bb2aa921d911 2024-11-15T17:01:02,746 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ce76bb7e61b4d159b65d18061afd15b to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/A/2ce76bb7e61b4d159b65d18061afd15b 2024-11-15T17:01:02,747 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a63b15efc32143c1be16f95bb97ee999 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/a63b15efc32143c1be16f95bb97ee999 2024-11-15T17:01:02,748 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/be80309ed2f042da811c0360cbb338e2 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/B/be80309ed2f042da811c0360cbb338e2 2024-11-15T17:01:02,750 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8bcd2dcd2c934d83ab1bf2cc0b738d93 to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/8bcd2dcd2c934d83ab1bf2cc0b738d93 2024-11-15T17:01:02,750 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe374ed3054c4f358d266355be5191fa to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/C/fe374ed3054c4f358d266355be5191fa 2024-11-15T17:01:02,752 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/recovered.edits/362.seqid to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e/recovered.edits/362.seqid 2024-11-15T17:01:02,753 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/default/TestAcidGuarantees/696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,753 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:01:02,753 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:01:02,754 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T17:01:02,755 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111508c6723cfd1e4d4991214b2520c3d3aa_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111508c6723cfd1e4d4991214b2520c3d3aa_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,756 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151196b049830c457ca1c840e8ef138283_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151196b049830c457ca1c840e8ef138283_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,757 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a18f0e0fe0046bdaf7d39fb7392cb6f_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a18f0e0fe0046bdaf7d39fb7392cb6f_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,758 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c3b7de8e40949f8a6ecbcc2afd411c9_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c3b7de8e40949f8a6ecbcc2afd411c9_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,759 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c7da72964ca438383dac56c742a71ce_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c7da72964ca438383dac56c742a71ce_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,759 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155499068c441d456e9c6ce0121e8dada9_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155499068c441d456e9c6ce0121e8dada9_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111571163ca99b9e451f8cffc001f16445e3_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111571163ca99b9e451f8cffc001f16445e3_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111572353f280e48422d926168e48f711a56_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111572353f280e48422d926168e48f711a56_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,762 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579323a253ca64ffc8d83e63647b921fc_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579323a253ca64ffc8d83e63647b921fc_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,763 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584430137c3b140d89e9107bb90df1ff4_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584430137c3b140d89e9107bb90df1ff4_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,764 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111596e820c14cd746899681245cb61e0d6c_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111596e820c14cd746899681245cb61e0d6c_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,765 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4387e6cc8b642b0a9e70cf9e3191b38_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4387e6cc8b642b0a9e70cf9e3191b38_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,766 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a6580becaef94750be152516617041d0_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a6580becaef94750be152516617041d0_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,767 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b481222c14e344cca54cf869176b79cb_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b481222c14e344cca54cf869176b79cb_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,767 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfe03a0fb2564d168c857605e54e53e9_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfe03a0fb2564d168c857605e54e53e9_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,768 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d66c1ec09def437e9d60c14155747eb7_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d66c1ec09def437e9d60c14155747eb7_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,769 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e65e823357d2473c830466e3da357e1e_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e65e823357d2473c830466e3da357e1e_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,770 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f72187573011466a98f9c9bf91da1465_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f72187573011466a98f9c9bf91da1465_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,771 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f89743d4f16746b4b999301e64575e25_696d98f18be792cdc1435b3b0e2fd21e to hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f89743d4f16746b4b999301e64575e25_696d98f18be792cdc1435b3b0e2fd21e 2024-11-15T17:01:02,771 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:01:02,773 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=178, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:01:02,775 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:01:02,777 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:01:02,777 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=178, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:01:02,777 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:01:02,778 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731690062777"}]},"ts":"9223372036854775807"} 2024-11-15T17:01:02,779 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:01:02,779 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 696d98f18be792cdc1435b3b0e2fd21e, NAME => 'TestAcidGuarantees,,1731690030901.696d98f18be792cdc1435b3b0e2fd21e.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:01:02,779 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:01:02,779 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731690062779"}]},"ts":"9223372036854775807"} 2024-11-15T17:01:02,780 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:01:02,781 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=178, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:01:02,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-15T17:01:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-15T17:01:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:01:02,846 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:01:02,855 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 236) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=340 (was 252) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4983 (was 5214) 2024-11-15T17:01:02,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T17:01:02,856 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:01:02,856 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:01:02,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:02,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:02,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:01:02,856 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T17:01:02,856 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1284221299, stopped=false 2024-11-15T17:01:02,857 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a9bde5b3a1b5,41785,1731689871395 2024-11-15T17:01:02,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T17:01:02,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:01:02,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T17:01:02,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:01:02,859 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T17:01:02,859 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:01:02,859 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:01:02,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:01:02,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:02,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:01:02,860 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a9bde5b3a1b5,35119,1731689872090' ***** 2024-11-15T17:01:02,860 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T17:01:02,860 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T17:01:02,861 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(959): stopping server a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a9bde5b3a1b5:35119. 2024-11-15T17:01:02,861 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:01:02,861 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T17:01:02,861 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T17:01:02,862 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T17:01:02,862 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T17:01:02,862 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:01:02,862 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T17:01:02,862 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T17:01:02,862 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T17:01:02,862 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T17:01:02,863 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T17:01:02,863 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T17:01:02,883 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/info/7509453fc1134737b893bc77aab57551 is 95, key is TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98./info:/1731689903081/DeleteFamily/seqid=0 2024-11-15T17:01:02,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742443_1619 (size=6514) 2024-11-15T17:01:02,887 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/info/7509453fc1134737b893bc77aab57551 2024-11-15T17:01:02,904 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/ns/f42a21e2dd1043ca88ce347d1effc95b is 93, key is TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98./ns:/1731689903033/DeleteFamily/seqid=0 2024-11-15T17:01:02,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742444_1620 (size=5906) 2024-11-15T17:01:02,949 INFO [regionserver/a9bde5b3a1b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T17:01:03,062 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:01:03,263 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:01:03,307 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/ns/f42a21e2dd1043ca88ce347d1effc95b 2024-11-15T17:01:03,326 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/rep_barrier/8918b1e8dcf44c4fbfe7129b4f92d179 is 102, key is TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98./rep_barrier:/1731689903033/DeleteFamily/seqid=0 2024-11-15T17:01:03,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742445_1621 (size=6025) 2024-11-15T17:01:03,463 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:01:03,524 INFO [regionserver/a9bde5b3a1b5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T17:01:03,524 INFO [regionserver/a9bde5b3a1b5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T17:01:03,663 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:01:03,729 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/rep_barrier/8918b1e8dcf44c4fbfe7129b4f92d179 2024-11-15T17:01:03,747 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/table/8af0f9ec9cfe47a79511be2b2fe45b30 is 96, key is TestAcidGuarantees,,1731689875986.1195534959da145ee3aa98da7f864a98./table:/1731689903033/DeleteFamily/seqid=0 2024-11-15T17:01:03,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742446_1622 (size=5936) 2024-11-15T17:01:03,863 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:01:04,063 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:01:04,151 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/table/8af0f9ec9cfe47a79511be2b2fe45b30 2024-11-15T17:01:04,154 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/info/7509453fc1134737b893bc77aab57551 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/info/7509453fc1134737b893bc77aab57551 2024-11-15T17:01:04,157 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/info/7509453fc1134737b893bc77aab57551, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T17:01:04,157 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/ns/f42a21e2dd1043ca88ce347d1effc95b as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/ns/f42a21e2dd1043ca88ce347d1effc95b 2024-11-15T17:01:04,160 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/ns/f42a21e2dd1043ca88ce347d1effc95b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T17:01:04,160 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/rep_barrier/8918b1e8dcf44c4fbfe7129b4f92d179 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/rep_barrier/8918b1e8dcf44c4fbfe7129b4f92d179 2024-11-15T17:01:04,163 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/rep_barrier/8918b1e8dcf44c4fbfe7129b4f92d179, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T17:01:04,164 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/.tmp/table/8af0f9ec9cfe47a79511be2b2fe45b30 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/table/8af0f9ec9cfe47a79511be2b2fe45b30 2024-11-15T17:01:04,166 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/table/8af0f9ec9cfe47a79511be2b2fe45b30, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T17:01:04,167 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1304ms, sequenceid=90, compaction requested=false 2024-11-15T17:01:04,170 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T17:01:04,171 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T17:01:04,171 INFO [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T17:01:04,171 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731690062862Running coprocessor pre-close hooks at 1731690062862Disabling compacts and flushes for region at 1731690062862Disabling writes for close at 1731690062862Obtaining lock to block concurrent updates at 1731690062863 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731690062863Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731690062863Flushing stores of hbase:meta,,1.1588230740 at 1731690062863Flushing 1588230740/info: creating writer at 1731690062864 (+1 ms)Flushing 1588230740/info: appending metadata at 1731690062883 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731690062883Flushing 1588230740/ns: creating writer at 1731690062890 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731690062903 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731690062903Flushing 1588230740/rep_barrier: creating writer at 1731690063310 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731690063326 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731690063326Flushing 1588230740/table: creating writer at 1731690063732 (+406 ms)Flushing 1588230740/table: appending metadata at 1731690063747 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731690063747Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e7cacee: reopening flushed file at 1731690064154 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f3027e8: reopening flushed file at 1731690064157 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd1a4c2: reopening flushed file at 1731690064160 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f96f133: reopening flushed file at 1731690064163 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1304ms, sequenceid=90, compaction requested=false at 1731690064167 (+4 ms)Writing region close event to WAL at 1731690064168 (+1 ms)Running coprocessor post-close hooks at 1731690064171 (+3 ms)Closed at 1731690064171 2024-11-15T17:01:04,171 DEBUG [RS_CLOSE_META-regionserver/a9bde5b3a1b5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T17:01:04,264 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(976): stopping server a9bde5b3a1b5,35119,1731689872090; all regions closed. 2024-11-15T17:01:04,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741834_1010 (size=25344) 2024-11-15T17:01:04,270 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/oldWALs 2024-11-15T17:01:04,270 INFO [RS:0;a9bde5b3a1b5:35119 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C35119%2C1731689872090.meta:.meta(num 1731689875578) 2024-11-15T17:01:04,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741832_1008 (size=13775690) 2024-11-15T17:01:04,274 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/oldWALs 2024-11-15T17:01:04,274 INFO [RS:0;a9bde5b3a1b5:35119 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a9bde5b3a1b5%2C35119%2C1731689872090:(num 1731689874624) 2024-11-15T17:01:04,274 DEBUG [RS:0;a9bde5b3a1b5:35119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:01:04,274 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T17:01:04,274 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T17:01:04,274 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.ChoreService(370): Chore service for: regionserver/a9bde5b3a1b5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T17:01:04,274 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T17:01:04,274 INFO [regionserver/a9bde5b3a1b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T17:01:04,275 INFO [RS:0;a9bde5b3a1b5:35119 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35119 2024-11-15T17:01:04,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a9bde5b3a1b5,35119,1731689872090 2024-11-15T17:01:04,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T17:01:04,280 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T17:01:04,280 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a9bde5b3a1b5,35119,1731689872090] 2024-11-15T17:01:04,282 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a9bde5b3a1b5,35119,1731689872090 already deleted, retry=false 2024-11-15T17:01:04,282 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a9bde5b3a1b5,35119,1731689872090 expired; onlineServers=0 2024-11-15T17:01:04,282 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a9bde5b3a1b5,41785,1731689871395' ***** 2024-11-15T17:01:04,282 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T17:01:04,282 INFO [M:0;a9bde5b3a1b5:41785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T17:01:04,282 INFO [M:0;a9bde5b3a1b5:41785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T17:01:04,282 DEBUG [M:0;a9bde5b3a1b5:41785 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T17:01:04,283 DEBUG [M:0;a9bde5b3a1b5:41785 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T17:01:04,283 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T17:01:04,283 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689874322 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.large.0-1731689874322,5,FailOnTimeoutGroup] 2024-11-15T17:01:04,283 DEBUG [master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689874331 {}] cleaner.HFileCleaner(306): Exit Thread[master/a9bde5b3a1b5:0:becomeActiveMaster-HFileCleaner.small.0-1731689874331,5,FailOnTimeoutGroup] 2024-11-15T17:01:04,283 INFO [M:0;a9bde5b3a1b5:41785 {}] hbase.ChoreService(370): Chore service for: master/a9bde5b3a1b5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T17:01:04,283 INFO [M:0;a9bde5b3a1b5:41785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T17:01:04,283 DEBUG [M:0;a9bde5b3a1b5:41785 {}] master.HMaster(1795): Stopping service threads 2024-11-15T17:01:04,283 INFO [M:0;a9bde5b3a1b5:41785 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T17:01:04,283 INFO [M:0;a9bde5b3a1b5:41785 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T17:01:04,283 ERROR [M:0;a9bde5b3a1b5:41785 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:45091 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45091,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T17:01:04,284 INFO [M:0;a9bde5b3a1b5:41785 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T17:01:04,284 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T17:01:04,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T17:01:04,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:01:04,285 DEBUG [M:0;a9bde5b3a1b5:41785 {}] zookeeper.ZKUtil(347): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T17:01:04,285 WARN [M:0;a9bde5b3a1b5:41785 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T17:01:04,285 INFO [M:0;a9bde5b3a1b5:41785 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/.lastflushedseqids 2024-11-15T17:01:04,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742447_1623 (size=72) 2024-11-15T17:01:04,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:01:04,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35119-0x101acddea460001, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:01:04,383 INFO [RS:0;a9bde5b3a1b5:35119 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T17:01:04,383 INFO [RS:0;a9bde5b3a1b5:35119 {}] regionserver.HRegionServer(1031): Exiting; stopping=a9bde5b3a1b5,35119,1731689872090; zookeeper connection closed. 2024-11-15T17:01:04,383 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75a89599 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75a89599 2024-11-15T17:01:04,384 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T17:01:04,698 INFO [M:0;a9bde5b3a1b5:41785 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T17:01:04,698 INFO [M:0;a9bde5b3a1b5:41785 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T17:01:04,698 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T17:01:04,698 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:01:04,698 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:01:04,698 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T17:01:04,698 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:01:04,698 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=801.94 KB heapSize=989.38 KB 2024-11-15T17:01:04,714 DEBUG [M:0;a9bde5b3a1b5:41785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f1b55f134264dc2a194237d890e03f2 is 82, key is hbase:meta,,1/info:regioninfo/1731689875666/Put/seqid=0 2024-11-15T17:01:04,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742448_1624 (size=5672) 2024-11-15T17:01:05,118 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2323 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f1b55f134264dc2a194237d890e03f2 2024-11-15T17:01:05,138 DEBUG [M:0;a9bde5b3a1b5:41785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2783378eb78646429b4bac121a795080 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x99/proc:d/1731690032972/Put/seqid=0 2024-11-15T17:01:05,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742449_1625 (size=45518) 2024-11-15T17:01:05,542 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=801.39 KB at sequenceid=2323 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2783378eb78646429b4bac121a795080 2024-11-15T17:01:05,545 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2783378eb78646429b4bac121a795080 2024-11-15T17:01:05,560 DEBUG [M:0;a9bde5b3a1b5:41785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b8f2cef09f3a47799621952bef5435b4 is 69, key is a9bde5b3a1b5,35119,1731689872090/rs:state/1731689874382/Put/seqid=0 2024-11-15T17:01:05,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073742450_1626 (size=5156) 2024-11-15T17:01:05,964 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2323 (bloomFilter=true), to=hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b8f2cef09f3a47799621952bef5435b4 2024-11-15T17:01:05,967 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f1b55f134264dc2a194237d890e03f2 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2f1b55f134264dc2a194237d890e03f2 2024-11-15T17:01:05,970 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2f1b55f134264dc2a194237d890e03f2, entries=8, sequenceid=2323, filesize=5.5 K 2024-11-15T17:01:05,970 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2783378eb78646429b4bac121a795080 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2783378eb78646429b4bac121a795080 2024-11-15T17:01:05,973 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2783378eb78646429b4bac121a795080 2024-11-15T17:01:05,974 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2783378eb78646429b4bac121a795080, entries=178, sequenceid=2323, filesize=44.5 K 2024-11-15T17:01:05,974 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b8f2cef09f3a47799621952bef5435b4 as hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b8f2cef09f3a47799621952bef5435b4 2024-11-15T17:01:05,977 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45091/user/jenkins/test-data/f9fd08fd-314d-3084-bdf2-2a05fa736f3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b8f2cef09f3a47799621952bef5435b4, entries=1, sequenceid=2323, filesize=5.0 K 2024-11-15T17:01:05,978 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(3140): Finished flush of dataSize ~801.94 KB/821189, heapSize ~989.08 KB/1012816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1280ms, sequenceid=2323, compaction requested=false 2024-11-15T17:01:05,979 INFO [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:01:05,979 DEBUG [M:0;a9bde5b3a1b5:41785 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731690064698Disabling compacts and flushes for region at 1731690064698Disabling writes for close at 1731690064698Obtaining lock to block concurrent updates at 1731690064698Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731690064698Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=821189, getHeapSize=1013056, getOffHeapSize=0, getCellsCount=2526 at 1731690064699 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731690064699Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731690064699Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731690064714 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731690064714Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731690065121 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731690065137 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731690065137Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731690065545 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731690065560 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731690065560Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30e6cd75: reopening flushed file at 1731690065967 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d7ae1a1: reopening flushed file at 1731690065970 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@342a80ee: reopening flushed file at 1731690065974 (+4 ms)Finished flush of dataSize ~801.94 KB/821189, heapSize ~989.08 KB/1012816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1280ms, sequenceid=2323, compaction requested=false at 1731690065978 (+4 ms)Writing region close event to WAL at 1731690065979 (+1 ms)Closed at 1731690065979 2024-11-15T17:01:05,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38441 is added to blk_1073741830_1006 (size=973415) 2024-11-15T17:01:05,982 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T17:01:05,982 INFO [M:0;a9bde5b3a1b5:41785 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T17:01:05,982 INFO [M:0;a9bde5b3a1b5:41785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41785 2024-11-15T17:01:05,983 INFO [M:0;a9bde5b3a1b5:41785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T17:01:06,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:01:06,087 INFO [M:0;a9bde5b3a1b5:41785 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T17:01:06,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41785-0x101acddea460000, quorum=127.0.0.1:49383, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:01:06,091 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T17:01:06,094 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T17:01:06,094 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T17:01:06,094 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T17:01:06,094 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/hadoop.log.dir/,STOPPED} 2024-11-15T17:01:06,097 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T17:01:06,097 WARN [BP-433987452-172.17.0.2-1731689868601 heartbeating to localhost/127.0.0.1:45091 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T17:01:06,097 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T17:01:06,097 WARN [BP-433987452-172.17.0.2-1731689868601 heartbeating to localhost/127.0.0.1:45091 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-433987452-172.17.0.2-1731689868601 (Datanode Uuid 75b1e1ef-6e8d-47b6-97f4-ab8c7454f9d7) service to localhost/127.0.0.1:45091 2024-11-15T17:01:06,099 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/data/data1/current/BP-433987452-172.17.0.2-1731689868601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T17:01:06,099 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/cluster_fd091a16-b3ac-f6f8-4422-fdbdda03be06/data/data2/current/BP-433987452-172.17.0.2-1731689868601 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T17:01:06,100 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T17:01:06,107 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T17:01:06,108 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T17:01:06,108 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T17:01:06,108 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T17:01:06,108 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9cceca13-214b-f56a-10ce-88e9482cd492/hadoop.log.dir/,STOPPED} 2024-11-15T17:01:06,123 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T17:01:06,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down